{ "base": { "dir": "~/AI/local_models_exl/glm-4.6-exl3-6bpw", "bpw": 6.015378480832238 }, "alts": [ { "dir": "~/AI/local_models_exl/glm-4.6-exl3-8bpw", "bpw": 8.014963334231846 } ], "groups": [ { "idx": 0, "layers": [ "model.layers.0.self_attn.q_proj" ], "candidates": [ { "dkld": -1.4812301378697179e-05, "dbits": 125829120 } ] }, { "idx": 1, "layers": [ "model.layers.0.self_attn.k_proj", "model.layers.0.self_attn.v_proj" ], "candidates": [ { "dkld": 2.4859455879777627e-05, "dbits": 20971520 } ] }, { "idx": 2, "layers": [ "model.layers.0.self_attn.o_proj" ], "candidates": [ { "dkld": -2.515355299692601e-05, "dbits": 125829120 } ] }, { "idx": 3, "layers": [ "model.layers.0.mlp.gate_proj", "model.layers.0.mlp.up_proj" ], "candidates": [ { "dkld": 5.382613744586793e-06, "dbits": 251658240 } ] }, { "idx": 4, "layers": [ "model.layers.0.mlp.down_proj" ], "candidates": [ { "dkld": -2.1498906426131747e-05, "dbits": 125829120 } ] }, { "idx": 5, "layers": [ "model.layers.1.self_attn.q_proj" ], "candidates": [ { "dkld": 4.5831897296012315e-06, "dbits": 125829120 } ] }, { "idx": 6, "layers": [ "model.layers.1.self_attn.k_proj", "model.layers.1.self_attn.v_proj" ], "candidates": [ { "dkld": 4.058080958202525e-06, "dbits": 20971520 } ] }, { "idx": 7, "layers": [ "model.layers.1.self_attn.o_proj" ], "candidates": [ { "dkld": -1.7050906899385133e-05, "dbits": 125829120 } ] }, { "idx": 8, "layers": [ "model.layers.1.mlp.gate_proj", "model.layers.1.mlp.up_proj" ], "candidates": [ { "dkld": 2.4503789609298023e-05, "dbits": 251658240 } ] }, { "idx": 9, "layers": [ "model.layers.1.mlp.down_proj" ], "candidates": [ { "dkld": -5.0124214612878816e-05, "dbits": 125829120 } ] }, { "idx": 10, "layers": [ "model.layers.2.self_attn.q_proj" ], "candidates": [ { "dkld": -1.7609717906452737e-05, "dbits": 125829120 } ] }, { "idx": 11, "layers": [ "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj" ], "candidates": [ { "dkld": -1.1500174878165126e-05, "dbits": 20971520 } ] }, { "idx": 12, "layers": [ "model.layers.2.self_attn.o_proj" ], "candidates": [ { "dkld": 4.007286042906366e-06, "dbits": 125829120 } ] }, { "idx": 13, "layers": [ "model.layers.2.mlp.gate_proj", "model.layers.2.mlp.up_proj" ], "candidates": [ { "dkld": 4.400499165058136e-07, "dbits": 251658240 } ] }, { "idx": 14, "layers": [ "model.layers.2.mlp.down_proj" ], "candidates": [ { "dkld": -3.776809608098115e-05, "dbits": 125829120 } ] }, { "idx": 15, "layers": [ "model.layers.3.self_attn.q_proj" ], "candidates": [ { "dkld": -9.495648555457592e-06, "dbits": 125829120 } ] }, { "idx": 16, "layers": [ "model.layers.3.self_attn.k_proj", "model.layers.3.self_attn.v_proj" ], "candidates": [ { "dkld": -2.8721397393383064e-05, "dbits": 20971520 } ] }, { "idx": 17, "layers": [ "model.layers.3.self_attn.o_proj" ], "candidates": [ { "dkld": -2.344302483834324e-05, "dbits": 125829120 } ] }, { "idx": 18, "layers": [ "model.layers.3.mlp.shared_experts.gate_proj", "model.layers.3.mlp.shared_experts.up_proj", "model.layers.3.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 5.365459946915421e-06, "dbits": 47185920 } ] }, { "idx": 19, "layers": [ "model.layers.3.mlp.experts.0.gate_proj", "model.layers.3.mlp.experts.1.gate_proj", "model.layers.3.mlp.experts.2.gate_proj", "model.layers.3.mlp.experts.3.gate_proj", "model.layers.3.mlp.experts.4.gate_proj", "model.layers.3.mlp.experts.5.gate_proj", "model.layers.3.mlp.experts.6.gate_proj", "model.layers.3.mlp.experts.7.gate_proj", "model.layers.3.mlp.experts.8.gate_proj", "model.layers.3.mlp.experts.9.gate_proj", "model.layers.3.mlp.experts.10.gate_proj", "model.layers.3.mlp.experts.11.gate_proj", "model.layers.3.mlp.experts.12.gate_proj", "model.layers.3.mlp.experts.13.gate_proj", "model.layers.3.mlp.experts.14.gate_proj", "model.layers.3.mlp.experts.15.gate_proj", "model.layers.3.mlp.experts.16.gate_proj", "model.layers.3.mlp.experts.17.gate_proj", "model.layers.3.mlp.experts.18.gate_proj", "model.layers.3.mlp.experts.19.gate_proj", "model.layers.3.mlp.experts.20.gate_proj", "model.layers.3.mlp.experts.21.gate_proj", "model.layers.3.mlp.experts.22.gate_proj", "model.layers.3.mlp.experts.23.gate_proj", "model.layers.3.mlp.experts.24.gate_proj", "model.layers.3.mlp.experts.25.gate_proj", "model.layers.3.mlp.experts.26.gate_proj", "model.layers.3.mlp.experts.27.gate_proj", "model.layers.3.mlp.experts.28.gate_proj", "model.layers.3.mlp.experts.29.gate_proj", "model.layers.3.mlp.experts.30.gate_proj", "model.layers.3.mlp.experts.31.gate_proj", "model.layers.3.mlp.experts.32.gate_proj", "model.layers.3.mlp.experts.33.gate_proj", "model.layers.3.mlp.experts.34.gate_proj", "model.layers.3.mlp.experts.35.gate_proj", "model.layers.3.mlp.experts.36.gate_proj", "model.layers.3.mlp.experts.37.gate_proj", "model.layers.3.mlp.experts.38.gate_proj", "model.layers.3.mlp.experts.39.gate_proj", "model.layers.3.mlp.experts.40.gate_proj", "model.layers.3.mlp.experts.41.gate_proj", "model.layers.3.mlp.experts.42.gate_proj", "model.layers.3.mlp.experts.43.gate_proj", "model.layers.3.mlp.experts.44.gate_proj", "model.layers.3.mlp.experts.45.gate_proj", "model.layers.3.mlp.experts.46.gate_proj", "model.layers.3.mlp.experts.47.gate_proj", "model.layers.3.mlp.experts.48.gate_proj", "model.layers.3.mlp.experts.49.gate_proj", "model.layers.3.mlp.experts.50.gate_proj", "model.layers.3.mlp.experts.51.gate_proj", "model.layers.3.mlp.experts.52.gate_proj", "model.layers.3.mlp.experts.53.gate_proj", "model.layers.3.mlp.experts.54.gate_proj", "model.layers.3.mlp.experts.55.gate_proj", "model.layers.3.mlp.experts.56.gate_proj", "model.layers.3.mlp.experts.57.gate_proj", "model.layers.3.mlp.experts.58.gate_proj", "model.layers.3.mlp.experts.59.gate_proj", "model.layers.3.mlp.experts.60.gate_proj", "model.layers.3.mlp.experts.61.gate_proj", "model.layers.3.mlp.experts.62.gate_proj", "model.layers.3.mlp.experts.63.gate_proj", "model.layers.3.mlp.experts.64.gate_proj", "model.layers.3.mlp.experts.65.gate_proj", "model.layers.3.mlp.experts.66.gate_proj", "model.layers.3.mlp.experts.67.gate_proj", "model.layers.3.mlp.experts.68.gate_proj", "model.layers.3.mlp.experts.69.gate_proj", "model.layers.3.mlp.experts.70.gate_proj", "model.layers.3.mlp.experts.71.gate_proj", "model.layers.3.mlp.experts.72.gate_proj", "model.layers.3.mlp.experts.73.gate_proj", "model.layers.3.mlp.experts.74.gate_proj", "model.layers.3.mlp.experts.75.gate_proj", "model.layers.3.mlp.experts.76.gate_proj", "model.layers.3.mlp.experts.77.gate_proj", "model.layers.3.mlp.experts.78.gate_proj", "model.layers.3.mlp.experts.79.gate_proj", "model.layers.3.mlp.experts.80.gate_proj", "model.layers.3.mlp.experts.81.gate_proj", "model.layers.3.mlp.experts.82.gate_proj", "model.layers.3.mlp.experts.83.gate_proj", "model.layers.3.mlp.experts.84.gate_proj", "model.layers.3.mlp.experts.85.gate_proj", "model.layers.3.mlp.experts.86.gate_proj", "model.layers.3.mlp.experts.87.gate_proj", "model.layers.3.mlp.experts.88.gate_proj", "model.layers.3.mlp.experts.89.gate_proj", "model.layers.3.mlp.experts.90.gate_proj", "model.layers.3.mlp.experts.91.gate_proj", "model.layers.3.mlp.experts.92.gate_proj", "model.layers.3.mlp.experts.93.gate_proj", "model.layers.3.mlp.experts.94.gate_proj", "model.layers.3.mlp.experts.95.gate_proj", "model.layers.3.mlp.experts.96.gate_proj", "model.layers.3.mlp.experts.97.gate_proj", "model.layers.3.mlp.experts.98.gate_proj", "model.layers.3.mlp.experts.99.gate_proj", "model.layers.3.mlp.experts.100.gate_proj", "model.layers.3.mlp.experts.101.gate_proj", "model.layers.3.mlp.experts.102.gate_proj", "model.layers.3.mlp.experts.103.gate_proj", "model.layers.3.mlp.experts.104.gate_proj", "model.layers.3.mlp.experts.105.gate_proj", "model.layers.3.mlp.experts.106.gate_proj", "model.layers.3.mlp.experts.107.gate_proj", "model.layers.3.mlp.experts.108.gate_proj", "model.layers.3.mlp.experts.109.gate_proj", "model.layers.3.mlp.experts.110.gate_proj", "model.layers.3.mlp.experts.111.gate_proj", "model.layers.3.mlp.experts.112.gate_proj", "model.layers.3.mlp.experts.113.gate_proj", "model.layers.3.mlp.experts.114.gate_proj", "model.layers.3.mlp.experts.115.gate_proj", "model.layers.3.mlp.experts.116.gate_proj", "model.layers.3.mlp.experts.117.gate_proj", "model.layers.3.mlp.experts.118.gate_proj", "model.layers.3.mlp.experts.119.gate_proj", "model.layers.3.mlp.experts.120.gate_proj", "model.layers.3.mlp.experts.121.gate_proj", "model.layers.3.mlp.experts.122.gate_proj", "model.layers.3.mlp.experts.123.gate_proj", "model.layers.3.mlp.experts.124.gate_proj", "model.layers.3.mlp.experts.125.gate_proj", "model.layers.3.mlp.experts.126.gate_proj", "model.layers.3.mlp.experts.127.gate_proj", "model.layers.3.mlp.experts.128.gate_proj", "model.layers.3.mlp.experts.129.gate_proj", "model.layers.3.mlp.experts.130.gate_proj", "model.layers.3.mlp.experts.131.gate_proj", "model.layers.3.mlp.experts.132.gate_proj", "model.layers.3.mlp.experts.133.gate_proj", "model.layers.3.mlp.experts.134.gate_proj", "model.layers.3.mlp.experts.135.gate_proj", "model.layers.3.mlp.experts.136.gate_proj", "model.layers.3.mlp.experts.137.gate_proj", "model.layers.3.mlp.experts.138.gate_proj", "model.layers.3.mlp.experts.139.gate_proj", "model.layers.3.mlp.experts.140.gate_proj", "model.layers.3.mlp.experts.141.gate_proj", "model.layers.3.mlp.experts.142.gate_proj", "model.layers.3.mlp.experts.143.gate_proj", "model.layers.3.mlp.experts.144.gate_proj", "model.layers.3.mlp.experts.145.gate_proj", "model.layers.3.mlp.experts.146.gate_proj", "model.layers.3.mlp.experts.147.gate_proj", "model.layers.3.mlp.experts.148.gate_proj", "model.layers.3.mlp.experts.149.gate_proj", "model.layers.3.mlp.experts.150.gate_proj", "model.layers.3.mlp.experts.151.gate_proj", "model.layers.3.mlp.experts.152.gate_proj", "model.layers.3.mlp.experts.153.gate_proj", "model.layers.3.mlp.experts.154.gate_proj", "model.layers.3.mlp.experts.155.gate_proj", "model.layers.3.mlp.experts.156.gate_proj", "model.layers.3.mlp.experts.157.gate_proj", "model.layers.3.mlp.experts.158.gate_proj", "model.layers.3.mlp.experts.159.gate_proj", "model.layers.3.mlp.experts.0.up_proj", "model.layers.3.mlp.experts.1.up_proj", "model.layers.3.mlp.experts.2.up_proj", "model.layers.3.mlp.experts.3.up_proj", "model.layers.3.mlp.experts.4.up_proj", "model.layers.3.mlp.experts.5.up_proj", "model.layers.3.mlp.experts.6.up_proj", "model.layers.3.mlp.experts.7.up_proj", "model.layers.3.mlp.experts.8.up_proj", "model.layers.3.mlp.experts.9.up_proj", "model.layers.3.mlp.experts.10.up_proj", "model.layers.3.mlp.experts.11.up_proj", "model.layers.3.mlp.experts.12.up_proj", "model.layers.3.mlp.experts.13.up_proj", "model.layers.3.mlp.experts.14.up_proj", "model.layers.3.mlp.experts.15.up_proj", "model.layers.3.mlp.experts.16.up_proj", "model.layers.3.mlp.experts.17.up_proj", "model.layers.3.mlp.experts.18.up_proj", "model.layers.3.mlp.experts.19.up_proj", "model.layers.3.mlp.experts.20.up_proj", "model.layers.3.mlp.experts.21.up_proj", "model.layers.3.mlp.experts.22.up_proj", "model.layers.3.mlp.experts.23.up_proj", "model.layers.3.mlp.experts.24.up_proj", "model.layers.3.mlp.experts.25.up_proj", "model.layers.3.mlp.experts.26.up_proj", "model.layers.3.mlp.experts.27.up_proj", "model.layers.3.mlp.experts.28.up_proj", "model.layers.3.mlp.experts.29.up_proj", "model.layers.3.mlp.experts.30.up_proj", "model.layers.3.mlp.experts.31.up_proj", "model.layers.3.mlp.experts.32.up_proj", "model.layers.3.mlp.experts.33.up_proj", "model.layers.3.mlp.experts.34.up_proj", "model.layers.3.mlp.experts.35.up_proj", "model.layers.3.mlp.experts.36.up_proj", "model.layers.3.mlp.experts.37.up_proj", "model.layers.3.mlp.experts.38.up_proj", "model.layers.3.mlp.experts.39.up_proj", "model.layers.3.mlp.experts.40.up_proj", "model.layers.3.mlp.experts.41.up_proj", "model.layers.3.mlp.experts.42.up_proj", "model.layers.3.mlp.experts.43.up_proj", "model.layers.3.mlp.experts.44.up_proj", "model.layers.3.mlp.experts.45.up_proj", "model.layers.3.mlp.experts.46.up_proj", "model.layers.3.mlp.experts.47.up_proj", "model.layers.3.mlp.experts.48.up_proj", "model.layers.3.mlp.experts.49.up_proj", "model.layers.3.mlp.experts.50.up_proj", "model.layers.3.mlp.experts.51.up_proj", "model.layers.3.mlp.experts.52.up_proj", "model.layers.3.mlp.experts.53.up_proj", "model.layers.3.mlp.experts.54.up_proj", "model.layers.3.mlp.experts.55.up_proj", "model.layers.3.mlp.experts.56.up_proj", "model.layers.3.mlp.experts.57.up_proj", "model.layers.3.mlp.experts.58.up_proj", "model.layers.3.mlp.experts.59.up_proj", "model.layers.3.mlp.experts.60.up_proj", "model.layers.3.mlp.experts.61.up_proj", "model.layers.3.mlp.experts.62.up_proj", "model.layers.3.mlp.experts.63.up_proj", "model.layers.3.mlp.experts.64.up_proj", "model.layers.3.mlp.experts.65.up_proj", "model.layers.3.mlp.experts.66.up_proj", "model.layers.3.mlp.experts.67.up_proj", "model.layers.3.mlp.experts.68.up_proj", "model.layers.3.mlp.experts.69.up_proj", "model.layers.3.mlp.experts.70.up_proj", "model.layers.3.mlp.experts.71.up_proj", "model.layers.3.mlp.experts.72.up_proj", "model.layers.3.mlp.experts.73.up_proj", "model.layers.3.mlp.experts.74.up_proj", "model.layers.3.mlp.experts.75.up_proj", "model.layers.3.mlp.experts.76.up_proj", "model.layers.3.mlp.experts.77.up_proj", "model.layers.3.mlp.experts.78.up_proj", "model.layers.3.mlp.experts.79.up_proj", "model.layers.3.mlp.experts.80.up_proj", "model.layers.3.mlp.experts.81.up_proj", "model.layers.3.mlp.experts.82.up_proj", "model.layers.3.mlp.experts.83.up_proj", "model.layers.3.mlp.experts.84.up_proj", "model.layers.3.mlp.experts.85.up_proj", "model.layers.3.mlp.experts.86.up_proj", "model.layers.3.mlp.experts.87.up_proj", "model.layers.3.mlp.experts.88.up_proj", "model.layers.3.mlp.experts.89.up_proj", "model.layers.3.mlp.experts.90.up_proj", "model.layers.3.mlp.experts.91.up_proj", "model.layers.3.mlp.experts.92.up_proj", "model.layers.3.mlp.experts.93.up_proj", "model.layers.3.mlp.experts.94.up_proj", "model.layers.3.mlp.experts.95.up_proj", "model.layers.3.mlp.experts.96.up_proj", "model.layers.3.mlp.experts.97.up_proj", "model.layers.3.mlp.experts.98.up_proj", "model.layers.3.mlp.experts.99.up_proj", "model.layers.3.mlp.experts.100.up_proj", "model.layers.3.mlp.experts.101.up_proj", "model.layers.3.mlp.experts.102.up_proj", "model.layers.3.mlp.experts.103.up_proj", "model.layers.3.mlp.experts.104.up_proj", "model.layers.3.mlp.experts.105.up_proj", "model.layers.3.mlp.experts.106.up_proj", "model.layers.3.mlp.experts.107.up_proj", "model.layers.3.mlp.experts.108.up_proj", "model.layers.3.mlp.experts.109.up_proj", "model.layers.3.mlp.experts.110.up_proj", "model.layers.3.mlp.experts.111.up_proj", "model.layers.3.mlp.experts.112.up_proj", "model.layers.3.mlp.experts.113.up_proj", "model.layers.3.mlp.experts.114.up_proj", "model.layers.3.mlp.experts.115.up_proj", "model.layers.3.mlp.experts.116.up_proj", "model.layers.3.mlp.experts.117.up_proj", "model.layers.3.mlp.experts.118.up_proj", "model.layers.3.mlp.experts.119.up_proj", "model.layers.3.mlp.experts.120.up_proj", "model.layers.3.mlp.experts.121.up_proj", "model.layers.3.mlp.experts.122.up_proj", "model.layers.3.mlp.experts.123.up_proj", "model.layers.3.mlp.experts.124.up_proj", "model.layers.3.mlp.experts.125.up_proj", "model.layers.3.mlp.experts.126.up_proj", "model.layers.3.mlp.experts.127.up_proj", "model.layers.3.mlp.experts.128.up_proj", "model.layers.3.mlp.experts.129.up_proj", "model.layers.3.mlp.experts.130.up_proj", "model.layers.3.mlp.experts.131.up_proj", "model.layers.3.mlp.experts.132.up_proj", "model.layers.3.mlp.experts.133.up_proj", "model.layers.3.mlp.experts.134.up_proj", "model.layers.3.mlp.experts.135.up_proj", "model.layers.3.mlp.experts.136.up_proj", "model.layers.3.mlp.experts.137.up_proj", "model.layers.3.mlp.experts.138.up_proj", "model.layers.3.mlp.experts.139.up_proj", "model.layers.3.mlp.experts.140.up_proj", "model.layers.3.mlp.experts.141.up_proj", "model.layers.3.mlp.experts.142.up_proj", "model.layers.3.mlp.experts.143.up_proj", "model.layers.3.mlp.experts.144.up_proj", "model.layers.3.mlp.experts.145.up_proj", "model.layers.3.mlp.experts.146.up_proj", "model.layers.3.mlp.experts.147.up_proj", "model.layers.3.mlp.experts.148.up_proj", "model.layers.3.mlp.experts.149.up_proj", "model.layers.3.mlp.experts.150.up_proj", "model.layers.3.mlp.experts.151.up_proj", "model.layers.3.mlp.experts.152.up_proj", "model.layers.3.mlp.experts.153.up_proj", "model.layers.3.mlp.experts.154.up_proj", "model.layers.3.mlp.experts.155.up_proj", "model.layers.3.mlp.experts.156.up_proj", "model.layers.3.mlp.experts.157.up_proj", "model.layers.3.mlp.experts.158.up_proj", "model.layers.3.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.949554568156557e-06, "dbits": 5033164800 } ] }, { "idx": 20, "layers": [ "model.layers.3.mlp.experts.0.down_proj", "model.layers.3.mlp.experts.1.down_proj", "model.layers.3.mlp.experts.2.down_proj", "model.layers.3.mlp.experts.3.down_proj", "model.layers.3.mlp.experts.4.down_proj", "model.layers.3.mlp.experts.5.down_proj", "model.layers.3.mlp.experts.6.down_proj", "model.layers.3.mlp.experts.7.down_proj", "model.layers.3.mlp.experts.8.down_proj", "model.layers.3.mlp.experts.9.down_proj", "model.layers.3.mlp.experts.10.down_proj", "model.layers.3.mlp.experts.11.down_proj", "model.layers.3.mlp.experts.12.down_proj", "model.layers.3.mlp.experts.13.down_proj", "model.layers.3.mlp.experts.14.down_proj", "model.layers.3.mlp.experts.15.down_proj", "model.layers.3.mlp.experts.16.down_proj", "model.layers.3.mlp.experts.17.down_proj", "model.layers.3.mlp.experts.18.down_proj", "model.layers.3.mlp.experts.19.down_proj", "model.layers.3.mlp.experts.20.down_proj", "model.layers.3.mlp.experts.21.down_proj", "model.layers.3.mlp.experts.22.down_proj", "model.layers.3.mlp.experts.23.down_proj", "model.layers.3.mlp.experts.24.down_proj", "model.layers.3.mlp.experts.25.down_proj", "model.layers.3.mlp.experts.26.down_proj", "model.layers.3.mlp.experts.27.down_proj", "model.layers.3.mlp.experts.28.down_proj", "model.layers.3.mlp.experts.29.down_proj", "model.layers.3.mlp.experts.30.down_proj", "model.layers.3.mlp.experts.31.down_proj", "model.layers.3.mlp.experts.32.down_proj", "model.layers.3.mlp.experts.33.down_proj", "model.layers.3.mlp.experts.34.down_proj", "model.layers.3.mlp.experts.35.down_proj", "model.layers.3.mlp.experts.36.down_proj", "model.layers.3.mlp.experts.37.down_proj", "model.layers.3.mlp.experts.38.down_proj", "model.layers.3.mlp.experts.39.down_proj", "model.layers.3.mlp.experts.40.down_proj", "model.layers.3.mlp.experts.41.down_proj", "model.layers.3.mlp.experts.42.down_proj", "model.layers.3.mlp.experts.43.down_proj", "model.layers.3.mlp.experts.44.down_proj", "model.layers.3.mlp.experts.45.down_proj", "model.layers.3.mlp.experts.46.down_proj", "model.layers.3.mlp.experts.47.down_proj", "model.layers.3.mlp.experts.48.down_proj", "model.layers.3.mlp.experts.49.down_proj", "model.layers.3.mlp.experts.50.down_proj", "model.layers.3.mlp.experts.51.down_proj", "model.layers.3.mlp.experts.52.down_proj", "model.layers.3.mlp.experts.53.down_proj", "model.layers.3.mlp.experts.54.down_proj", "model.layers.3.mlp.experts.55.down_proj", "model.layers.3.mlp.experts.56.down_proj", "model.layers.3.mlp.experts.57.down_proj", "model.layers.3.mlp.experts.58.down_proj", "model.layers.3.mlp.experts.59.down_proj", "model.layers.3.mlp.experts.60.down_proj", "model.layers.3.mlp.experts.61.down_proj", "model.layers.3.mlp.experts.62.down_proj", "model.layers.3.mlp.experts.63.down_proj", "model.layers.3.mlp.experts.64.down_proj", "model.layers.3.mlp.experts.65.down_proj", "model.layers.3.mlp.experts.66.down_proj", "model.layers.3.mlp.experts.67.down_proj", "model.layers.3.mlp.experts.68.down_proj", "model.layers.3.mlp.experts.69.down_proj", "model.layers.3.mlp.experts.70.down_proj", "model.layers.3.mlp.experts.71.down_proj", "model.layers.3.mlp.experts.72.down_proj", "model.layers.3.mlp.experts.73.down_proj", "model.layers.3.mlp.experts.74.down_proj", "model.layers.3.mlp.experts.75.down_proj", "model.layers.3.mlp.experts.76.down_proj", "model.layers.3.mlp.experts.77.down_proj", "model.layers.3.mlp.experts.78.down_proj", "model.layers.3.mlp.experts.79.down_proj", "model.layers.3.mlp.experts.80.down_proj", "model.layers.3.mlp.experts.81.down_proj", "model.layers.3.mlp.experts.82.down_proj", "model.layers.3.mlp.experts.83.down_proj", "model.layers.3.mlp.experts.84.down_proj", "model.layers.3.mlp.experts.85.down_proj", "model.layers.3.mlp.experts.86.down_proj", "model.layers.3.mlp.experts.87.down_proj", "model.layers.3.mlp.experts.88.down_proj", "model.layers.3.mlp.experts.89.down_proj", "model.layers.3.mlp.experts.90.down_proj", "model.layers.3.mlp.experts.91.down_proj", "model.layers.3.mlp.experts.92.down_proj", "model.layers.3.mlp.experts.93.down_proj", "model.layers.3.mlp.experts.94.down_proj", "model.layers.3.mlp.experts.95.down_proj", "model.layers.3.mlp.experts.96.down_proj", "model.layers.3.mlp.experts.97.down_proj", "model.layers.3.mlp.experts.98.down_proj", "model.layers.3.mlp.experts.99.down_proj", "model.layers.3.mlp.experts.100.down_proj", "model.layers.3.mlp.experts.101.down_proj", "model.layers.3.mlp.experts.102.down_proj", "model.layers.3.mlp.experts.103.down_proj", "model.layers.3.mlp.experts.104.down_proj", "model.layers.3.mlp.experts.105.down_proj", "model.layers.3.mlp.experts.106.down_proj", "model.layers.3.mlp.experts.107.down_proj", "model.layers.3.mlp.experts.108.down_proj", "model.layers.3.mlp.experts.109.down_proj", "model.layers.3.mlp.experts.110.down_proj", "model.layers.3.mlp.experts.111.down_proj", "model.layers.3.mlp.experts.112.down_proj", "model.layers.3.mlp.experts.113.down_proj", "model.layers.3.mlp.experts.114.down_proj", "model.layers.3.mlp.experts.115.down_proj", "model.layers.3.mlp.experts.116.down_proj", "model.layers.3.mlp.experts.117.down_proj", "model.layers.3.mlp.experts.118.down_proj", "model.layers.3.mlp.experts.119.down_proj", "model.layers.3.mlp.experts.120.down_proj", "model.layers.3.mlp.experts.121.down_proj", "model.layers.3.mlp.experts.122.down_proj", "model.layers.3.mlp.experts.123.down_proj", "model.layers.3.mlp.experts.124.down_proj", "model.layers.3.mlp.experts.125.down_proj", "model.layers.3.mlp.experts.126.down_proj", "model.layers.3.mlp.experts.127.down_proj", "model.layers.3.mlp.experts.128.down_proj", "model.layers.3.mlp.experts.129.down_proj", "model.layers.3.mlp.experts.130.down_proj", "model.layers.3.mlp.experts.131.down_proj", "model.layers.3.mlp.experts.132.down_proj", "model.layers.3.mlp.experts.133.down_proj", "model.layers.3.mlp.experts.134.down_proj", "model.layers.3.mlp.experts.135.down_proj", "model.layers.3.mlp.experts.136.down_proj", "model.layers.3.mlp.experts.137.down_proj", "model.layers.3.mlp.experts.138.down_proj", "model.layers.3.mlp.experts.139.down_proj", "model.layers.3.mlp.experts.140.down_proj", "model.layers.3.mlp.experts.141.down_proj", "model.layers.3.mlp.experts.142.down_proj", "model.layers.3.mlp.experts.143.down_proj", "model.layers.3.mlp.experts.144.down_proj", "model.layers.3.mlp.experts.145.down_proj", "model.layers.3.mlp.experts.146.down_proj", "model.layers.3.mlp.experts.147.down_proj", "model.layers.3.mlp.experts.148.down_proj", "model.layers.3.mlp.experts.149.down_proj", "model.layers.3.mlp.experts.150.down_proj", "model.layers.3.mlp.experts.151.down_proj", "model.layers.3.mlp.experts.152.down_proj", "model.layers.3.mlp.experts.153.down_proj", "model.layers.3.mlp.experts.154.down_proj", "model.layers.3.mlp.experts.155.down_proj", "model.layers.3.mlp.experts.156.down_proj", "model.layers.3.mlp.experts.157.down_proj", "model.layers.3.mlp.experts.158.down_proj", "model.layers.3.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -3.6140001611784534e-06, "dbits": 2516582400 } ] }, { "idx": 21, "layers": [ "model.layers.4.self_attn.q_proj" ], "candidates": [ { "dkld": -2.0580284763127544e-05, "dbits": 125829120 } ] }, { "idx": 22, "layers": [ "model.layers.4.self_attn.k_proj", "model.layers.4.self_attn.v_proj" ], "candidates": [ { "dkld": 1.5421101124957204e-06, "dbits": 20971520 } ] }, { "idx": 23, "layers": [ "model.layers.4.self_attn.o_proj" ], "candidates": [ { "dkld": -8.456484647467776e-06, "dbits": 125829120 } ] }, { "idx": 24, "layers": [ "model.layers.4.mlp.shared_experts.gate_proj", "model.layers.4.mlp.shared_experts.up_proj", "model.layers.4.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.6120288637466685e-05, "dbits": 47185920 } ] }, { "idx": 25, "layers": [ "model.layers.4.mlp.experts.0.gate_proj", "model.layers.4.mlp.experts.1.gate_proj", "model.layers.4.mlp.experts.2.gate_proj", "model.layers.4.mlp.experts.3.gate_proj", "model.layers.4.mlp.experts.4.gate_proj", "model.layers.4.mlp.experts.5.gate_proj", "model.layers.4.mlp.experts.6.gate_proj", "model.layers.4.mlp.experts.7.gate_proj", "model.layers.4.mlp.experts.8.gate_proj", "model.layers.4.mlp.experts.9.gate_proj", "model.layers.4.mlp.experts.10.gate_proj", "model.layers.4.mlp.experts.11.gate_proj", "model.layers.4.mlp.experts.12.gate_proj", "model.layers.4.mlp.experts.13.gate_proj", "model.layers.4.mlp.experts.14.gate_proj", "model.layers.4.mlp.experts.15.gate_proj", "model.layers.4.mlp.experts.16.gate_proj", "model.layers.4.mlp.experts.17.gate_proj", "model.layers.4.mlp.experts.18.gate_proj", "model.layers.4.mlp.experts.19.gate_proj", "model.layers.4.mlp.experts.20.gate_proj", "model.layers.4.mlp.experts.21.gate_proj", "model.layers.4.mlp.experts.22.gate_proj", "model.layers.4.mlp.experts.23.gate_proj", "model.layers.4.mlp.experts.24.gate_proj", "model.layers.4.mlp.experts.25.gate_proj", "model.layers.4.mlp.experts.26.gate_proj", "model.layers.4.mlp.experts.27.gate_proj", "model.layers.4.mlp.experts.28.gate_proj", "model.layers.4.mlp.experts.29.gate_proj", "model.layers.4.mlp.experts.30.gate_proj", "model.layers.4.mlp.experts.31.gate_proj", "model.layers.4.mlp.experts.32.gate_proj", "model.layers.4.mlp.experts.33.gate_proj", "model.layers.4.mlp.experts.34.gate_proj", "model.layers.4.mlp.experts.35.gate_proj", "model.layers.4.mlp.experts.36.gate_proj", "model.layers.4.mlp.experts.37.gate_proj", "model.layers.4.mlp.experts.38.gate_proj", "model.layers.4.mlp.experts.39.gate_proj", "model.layers.4.mlp.experts.40.gate_proj", "model.layers.4.mlp.experts.41.gate_proj", "model.layers.4.mlp.experts.42.gate_proj", "model.layers.4.mlp.experts.43.gate_proj", "model.layers.4.mlp.experts.44.gate_proj", "model.layers.4.mlp.experts.45.gate_proj", "model.layers.4.mlp.experts.46.gate_proj", "model.layers.4.mlp.experts.47.gate_proj", "model.layers.4.mlp.experts.48.gate_proj", "model.layers.4.mlp.experts.49.gate_proj", "model.layers.4.mlp.experts.50.gate_proj", "model.layers.4.mlp.experts.51.gate_proj", "model.layers.4.mlp.experts.52.gate_proj", "model.layers.4.mlp.experts.53.gate_proj", "model.layers.4.mlp.experts.54.gate_proj", "model.layers.4.mlp.experts.55.gate_proj", "model.layers.4.mlp.experts.56.gate_proj", "model.layers.4.mlp.experts.57.gate_proj", "model.layers.4.mlp.experts.58.gate_proj", "model.layers.4.mlp.experts.59.gate_proj", "model.layers.4.mlp.experts.60.gate_proj", "model.layers.4.mlp.experts.61.gate_proj", "model.layers.4.mlp.experts.62.gate_proj", "model.layers.4.mlp.experts.63.gate_proj", "model.layers.4.mlp.experts.64.gate_proj", "model.layers.4.mlp.experts.65.gate_proj", "model.layers.4.mlp.experts.66.gate_proj", "model.layers.4.mlp.experts.67.gate_proj", "model.layers.4.mlp.experts.68.gate_proj", "model.layers.4.mlp.experts.69.gate_proj", "model.layers.4.mlp.experts.70.gate_proj", "model.layers.4.mlp.experts.71.gate_proj", "model.layers.4.mlp.experts.72.gate_proj", "model.layers.4.mlp.experts.73.gate_proj", "model.layers.4.mlp.experts.74.gate_proj", "model.layers.4.mlp.experts.75.gate_proj", "model.layers.4.mlp.experts.76.gate_proj", "model.layers.4.mlp.experts.77.gate_proj", "model.layers.4.mlp.experts.78.gate_proj", "model.layers.4.mlp.experts.79.gate_proj", "model.layers.4.mlp.experts.80.gate_proj", "model.layers.4.mlp.experts.81.gate_proj", "model.layers.4.mlp.experts.82.gate_proj", "model.layers.4.mlp.experts.83.gate_proj", "model.layers.4.mlp.experts.84.gate_proj", "model.layers.4.mlp.experts.85.gate_proj", "model.layers.4.mlp.experts.86.gate_proj", "model.layers.4.mlp.experts.87.gate_proj", "model.layers.4.mlp.experts.88.gate_proj", "model.layers.4.mlp.experts.89.gate_proj", "model.layers.4.mlp.experts.90.gate_proj", "model.layers.4.mlp.experts.91.gate_proj", "model.layers.4.mlp.experts.92.gate_proj", "model.layers.4.mlp.experts.93.gate_proj", "model.layers.4.mlp.experts.94.gate_proj", "model.layers.4.mlp.experts.95.gate_proj", "model.layers.4.mlp.experts.96.gate_proj", "model.layers.4.mlp.experts.97.gate_proj", "model.layers.4.mlp.experts.98.gate_proj", "model.layers.4.mlp.experts.99.gate_proj", "model.layers.4.mlp.experts.100.gate_proj", "model.layers.4.mlp.experts.101.gate_proj", "model.layers.4.mlp.experts.102.gate_proj", "model.layers.4.mlp.experts.103.gate_proj", "model.layers.4.mlp.experts.104.gate_proj", "model.layers.4.mlp.experts.105.gate_proj", "model.layers.4.mlp.experts.106.gate_proj", "model.layers.4.mlp.experts.107.gate_proj", "model.layers.4.mlp.experts.108.gate_proj", "model.layers.4.mlp.experts.109.gate_proj", "model.layers.4.mlp.experts.110.gate_proj", "model.layers.4.mlp.experts.111.gate_proj", "model.layers.4.mlp.experts.112.gate_proj", "model.layers.4.mlp.experts.113.gate_proj", "model.layers.4.mlp.experts.114.gate_proj", "model.layers.4.mlp.experts.115.gate_proj", "model.layers.4.mlp.experts.116.gate_proj", "model.layers.4.mlp.experts.117.gate_proj", "model.layers.4.mlp.experts.118.gate_proj", "model.layers.4.mlp.experts.119.gate_proj", "model.layers.4.mlp.experts.120.gate_proj", "model.layers.4.mlp.experts.121.gate_proj", "model.layers.4.mlp.experts.122.gate_proj", "model.layers.4.mlp.experts.123.gate_proj", "model.layers.4.mlp.experts.124.gate_proj", "model.layers.4.mlp.experts.125.gate_proj", "model.layers.4.mlp.experts.126.gate_proj", "model.layers.4.mlp.experts.127.gate_proj", "model.layers.4.mlp.experts.128.gate_proj", "model.layers.4.mlp.experts.129.gate_proj", "model.layers.4.mlp.experts.130.gate_proj", "model.layers.4.mlp.experts.131.gate_proj", "model.layers.4.mlp.experts.132.gate_proj", "model.layers.4.mlp.experts.133.gate_proj", "model.layers.4.mlp.experts.134.gate_proj", "model.layers.4.mlp.experts.135.gate_proj", "model.layers.4.mlp.experts.136.gate_proj", "model.layers.4.mlp.experts.137.gate_proj", "model.layers.4.mlp.experts.138.gate_proj", "model.layers.4.mlp.experts.139.gate_proj", "model.layers.4.mlp.experts.140.gate_proj", "model.layers.4.mlp.experts.141.gate_proj", "model.layers.4.mlp.experts.142.gate_proj", "model.layers.4.mlp.experts.143.gate_proj", "model.layers.4.mlp.experts.144.gate_proj", "model.layers.4.mlp.experts.145.gate_proj", "model.layers.4.mlp.experts.146.gate_proj", "model.layers.4.mlp.experts.147.gate_proj", "model.layers.4.mlp.experts.148.gate_proj", "model.layers.4.mlp.experts.149.gate_proj", "model.layers.4.mlp.experts.150.gate_proj", "model.layers.4.mlp.experts.151.gate_proj", "model.layers.4.mlp.experts.152.gate_proj", "model.layers.4.mlp.experts.153.gate_proj", "model.layers.4.mlp.experts.154.gate_proj", "model.layers.4.mlp.experts.155.gate_proj", "model.layers.4.mlp.experts.156.gate_proj", "model.layers.4.mlp.experts.157.gate_proj", "model.layers.4.mlp.experts.158.gate_proj", "model.layers.4.mlp.experts.159.gate_proj", "model.layers.4.mlp.experts.0.up_proj", "model.layers.4.mlp.experts.1.up_proj", "model.layers.4.mlp.experts.2.up_proj", "model.layers.4.mlp.experts.3.up_proj", "model.layers.4.mlp.experts.4.up_proj", "model.layers.4.mlp.experts.5.up_proj", "model.layers.4.mlp.experts.6.up_proj", "model.layers.4.mlp.experts.7.up_proj", "model.layers.4.mlp.experts.8.up_proj", "model.layers.4.mlp.experts.9.up_proj", "model.layers.4.mlp.experts.10.up_proj", "model.layers.4.mlp.experts.11.up_proj", "model.layers.4.mlp.experts.12.up_proj", "model.layers.4.mlp.experts.13.up_proj", "model.layers.4.mlp.experts.14.up_proj", "model.layers.4.mlp.experts.15.up_proj", "model.layers.4.mlp.experts.16.up_proj", "model.layers.4.mlp.experts.17.up_proj", "model.layers.4.mlp.experts.18.up_proj", "model.layers.4.mlp.experts.19.up_proj", "model.layers.4.mlp.experts.20.up_proj", "model.layers.4.mlp.experts.21.up_proj", "model.layers.4.mlp.experts.22.up_proj", "model.layers.4.mlp.experts.23.up_proj", "model.layers.4.mlp.experts.24.up_proj", "model.layers.4.mlp.experts.25.up_proj", "model.layers.4.mlp.experts.26.up_proj", "model.layers.4.mlp.experts.27.up_proj", "model.layers.4.mlp.experts.28.up_proj", "model.layers.4.mlp.experts.29.up_proj", "model.layers.4.mlp.experts.30.up_proj", "model.layers.4.mlp.experts.31.up_proj", "model.layers.4.mlp.experts.32.up_proj", "model.layers.4.mlp.experts.33.up_proj", "model.layers.4.mlp.experts.34.up_proj", "model.layers.4.mlp.experts.35.up_proj", "model.layers.4.mlp.experts.36.up_proj", "model.layers.4.mlp.experts.37.up_proj", "model.layers.4.mlp.experts.38.up_proj", "model.layers.4.mlp.experts.39.up_proj", "model.layers.4.mlp.experts.40.up_proj", "model.layers.4.mlp.experts.41.up_proj", "model.layers.4.mlp.experts.42.up_proj", "model.layers.4.mlp.experts.43.up_proj", "model.layers.4.mlp.experts.44.up_proj", "model.layers.4.mlp.experts.45.up_proj", "model.layers.4.mlp.experts.46.up_proj", "model.layers.4.mlp.experts.47.up_proj", "model.layers.4.mlp.experts.48.up_proj", "model.layers.4.mlp.experts.49.up_proj", "model.layers.4.mlp.experts.50.up_proj", "model.layers.4.mlp.experts.51.up_proj", "model.layers.4.mlp.experts.52.up_proj", "model.layers.4.mlp.experts.53.up_proj", "model.layers.4.mlp.experts.54.up_proj", "model.layers.4.mlp.experts.55.up_proj", "model.layers.4.mlp.experts.56.up_proj", "model.layers.4.mlp.experts.57.up_proj", "model.layers.4.mlp.experts.58.up_proj", "model.layers.4.mlp.experts.59.up_proj", "model.layers.4.mlp.experts.60.up_proj", "model.layers.4.mlp.experts.61.up_proj", "model.layers.4.mlp.experts.62.up_proj", "model.layers.4.mlp.experts.63.up_proj", "model.layers.4.mlp.experts.64.up_proj", "model.layers.4.mlp.experts.65.up_proj", "model.layers.4.mlp.experts.66.up_proj", "model.layers.4.mlp.experts.67.up_proj", "model.layers.4.mlp.experts.68.up_proj", "model.layers.4.mlp.experts.69.up_proj", "model.layers.4.mlp.experts.70.up_proj", "model.layers.4.mlp.experts.71.up_proj", "model.layers.4.mlp.experts.72.up_proj", "model.layers.4.mlp.experts.73.up_proj", "model.layers.4.mlp.experts.74.up_proj", "model.layers.4.mlp.experts.75.up_proj", "model.layers.4.mlp.experts.76.up_proj", "model.layers.4.mlp.experts.77.up_proj", "model.layers.4.mlp.experts.78.up_proj", "model.layers.4.mlp.experts.79.up_proj", "model.layers.4.mlp.experts.80.up_proj", "model.layers.4.mlp.experts.81.up_proj", "model.layers.4.mlp.experts.82.up_proj", "model.layers.4.mlp.experts.83.up_proj", "model.layers.4.mlp.experts.84.up_proj", "model.layers.4.mlp.experts.85.up_proj", "model.layers.4.mlp.experts.86.up_proj", "model.layers.4.mlp.experts.87.up_proj", "model.layers.4.mlp.experts.88.up_proj", "model.layers.4.mlp.experts.89.up_proj", "model.layers.4.mlp.experts.90.up_proj", "model.layers.4.mlp.experts.91.up_proj", "model.layers.4.mlp.experts.92.up_proj", "model.layers.4.mlp.experts.93.up_proj", "model.layers.4.mlp.experts.94.up_proj", "model.layers.4.mlp.experts.95.up_proj", "model.layers.4.mlp.experts.96.up_proj", "model.layers.4.mlp.experts.97.up_proj", "model.layers.4.mlp.experts.98.up_proj", "model.layers.4.mlp.experts.99.up_proj", "model.layers.4.mlp.experts.100.up_proj", "model.layers.4.mlp.experts.101.up_proj", "model.layers.4.mlp.experts.102.up_proj", "model.layers.4.mlp.experts.103.up_proj", "model.layers.4.mlp.experts.104.up_proj", "model.layers.4.mlp.experts.105.up_proj", "model.layers.4.mlp.experts.106.up_proj", "model.layers.4.mlp.experts.107.up_proj", "model.layers.4.mlp.experts.108.up_proj", "model.layers.4.mlp.experts.109.up_proj", "model.layers.4.mlp.experts.110.up_proj", "model.layers.4.mlp.experts.111.up_proj", "model.layers.4.mlp.experts.112.up_proj", "model.layers.4.mlp.experts.113.up_proj", "model.layers.4.mlp.experts.114.up_proj", "model.layers.4.mlp.experts.115.up_proj", "model.layers.4.mlp.experts.116.up_proj", "model.layers.4.mlp.experts.117.up_proj", "model.layers.4.mlp.experts.118.up_proj", "model.layers.4.mlp.experts.119.up_proj", "model.layers.4.mlp.experts.120.up_proj", "model.layers.4.mlp.experts.121.up_proj", "model.layers.4.mlp.experts.122.up_proj", "model.layers.4.mlp.experts.123.up_proj", "model.layers.4.mlp.experts.124.up_proj", "model.layers.4.mlp.experts.125.up_proj", "model.layers.4.mlp.experts.126.up_proj", "model.layers.4.mlp.experts.127.up_proj", "model.layers.4.mlp.experts.128.up_proj", "model.layers.4.mlp.experts.129.up_proj", "model.layers.4.mlp.experts.130.up_proj", "model.layers.4.mlp.experts.131.up_proj", "model.layers.4.mlp.experts.132.up_proj", "model.layers.4.mlp.experts.133.up_proj", "model.layers.4.mlp.experts.134.up_proj", "model.layers.4.mlp.experts.135.up_proj", "model.layers.4.mlp.experts.136.up_proj", "model.layers.4.mlp.experts.137.up_proj", "model.layers.4.mlp.experts.138.up_proj", "model.layers.4.mlp.experts.139.up_proj", "model.layers.4.mlp.experts.140.up_proj", "model.layers.4.mlp.experts.141.up_proj", "model.layers.4.mlp.experts.142.up_proj", "model.layers.4.mlp.experts.143.up_proj", "model.layers.4.mlp.experts.144.up_proj", "model.layers.4.mlp.experts.145.up_proj", "model.layers.4.mlp.experts.146.up_proj", "model.layers.4.mlp.experts.147.up_proj", "model.layers.4.mlp.experts.148.up_proj", "model.layers.4.mlp.experts.149.up_proj", "model.layers.4.mlp.experts.150.up_proj", "model.layers.4.mlp.experts.151.up_proj", "model.layers.4.mlp.experts.152.up_proj", "model.layers.4.mlp.experts.153.up_proj", "model.layers.4.mlp.experts.154.up_proj", "model.layers.4.mlp.experts.155.up_proj", "model.layers.4.mlp.experts.156.up_proj", "model.layers.4.mlp.experts.157.up_proj", "model.layers.4.mlp.experts.158.up_proj", "model.layers.4.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -8.156136027537265e-06, "dbits": 5033164800 } ] }, { "idx": 26, "layers": [ "model.layers.4.mlp.experts.0.down_proj", "model.layers.4.mlp.experts.1.down_proj", "model.layers.4.mlp.experts.2.down_proj", "model.layers.4.mlp.experts.3.down_proj", "model.layers.4.mlp.experts.4.down_proj", "model.layers.4.mlp.experts.5.down_proj", "model.layers.4.mlp.experts.6.down_proj", "model.layers.4.mlp.experts.7.down_proj", "model.layers.4.mlp.experts.8.down_proj", "model.layers.4.mlp.experts.9.down_proj", "model.layers.4.mlp.experts.10.down_proj", "model.layers.4.mlp.experts.11.down_proj", "model.layers.4.mlp.experts.12.down_proj", "model.layers.4.mlp.experts.13.down_proj", "model.layers.4.mlp.experts.14.down_proj", "model.layers.4.mlp.experts.15.down_proj", "model.layers.4.mlp.experts.16.down_proj", "model.layers.4.mlp.experts.17.down_proj", "model.layers.4.mlp.experts.18.down_proj", "model.layers.4.mlp.experts.19.down_proj", "model.layers.4.mlp.experts.20.down_proj", "model.layers.4.mlp.experts.21.down_proj", "model.layers.4.mlp.experts.22.down_proj", "model.layers.4.mlp.experts.23.down_proj", "model.layers.4.mlp.experts.24.down_proj", "model.layers.4.mlp.experts.25.down_proj", "model.layers.4.mlp.experts.26.down_proj", "model.layers.4.mlp.experts.27.down_proj", "model.layers.4.mlp.experts.28.down_proj", "model.layers.4.mlp.experts.29.down_proj", "model.layers.4.mlp.experts.30.down_proj", "model.layers.4.mlp.experts.31.down_proj", "model.layers.4.mlp.experts.32.down_proj", "model.layers.4.mlp.experts.33.down_proj", "model.layers.4.mlp.experts.34.down_proj", "model.layers.4.mlp.experts.35.down_proj", "model.layers.4.mlp.experts.36.down_proj", "model.layers.4.mlp.experts.37.down_proj", "model.layers.4.mlp.experts.38.down_proj", "model.layers.4.mlp.experts.39.down_proj", "model.layers.4.mlp.experts.40.down_proj", "model.layers.4.mlp.experts.41.down_proj", "model.layers.4.mlp.experts.42.down_proj", "model.layers.4.mlp.experts.43.down_proj", "model.layers.4.mlp.experts.44.down_proj", "model.layers.4.mlp.experts.45.down_proj", "model.layers.4.mlp.experts.46.down_proj", "model.layers.4.mlp.experts.47.down_proj", "model.layers.4.mlp.experts.48.down_proj", "model.layers.4.mlp.experts.49.down_proj", "model.layers.4.mlp.experts.50.down_proj", "model.layers.4.mlp.experts.51.down_proj", "model.layers.4.mlp.experts.52.down_proj", "model.layers.4.mlp.experts.53.down_proj", "model.layers.4.mlp.experts.54.down_proj", "model.layers.4.mlp.experts.55.down_proj", "model.layers.4.mlp.experts.56.down_proj", "model.layers.4.mlp.experts.57.down_proj", "model.layers.4.mlp.experts.58.down_proj", "model.layers.4.mlp.experts.59.down_proj", "model.layers.4.mlp.experts.60.down_proj", "model.layers.4.mlp.experts.61.down_proj", "model.layers.4.mlp.experts.62.down_proj", "model.layers.4.mlp.experts.63.down_proj", "model.layers.4.mlp.experts.64.down_proj", "model.layers.4.mlp.experts.65.down_proj", "model.layers.4.mlp.experts.66.down_proj", "model.layers.4.mlp.experts.67.down_proj", "model.layers.4.mlp.experts.68.down_proj", "model.layers.4.mlp.experts.69.down_proj", "model.layers.4.mlp.experts.70.down_proj", "model.layers.4.mlp.experts.71.down_proj", "model.layers.4.mlp.experts.72.down_proj", "model.layers.4.mlp.experts.73.down_proj", "model.layers.4.mlp.experts.74.down_proj", "model.layers.4.mlp.experts.75.down_proj", "model.layers.4.mlp.experts.76.down_proj", "model.layers.4.mlp.experts.77.down_proj", "model.layers.4.mlp.experts.78.down_proj", "model.layers.4.mlp.experts.79.down_proj", "model.layers.4.mlp.experts.80.down_proj", "model.layers.4.mlp.experts.81.down_proj", "model.layers.4.mlp.experts.82.down_proj", "model.layers.4.mlp.experts.83.down_proj", "model.layers.4.mlp.experts.84.down_proj", "model.layers.4.mlp.experts.85.down_proj", "model.layers.4.mlp.experts.86.down_proj", "model.layers.4.mlp.experts.87.down_proj", "model.layers.4.mlp.experts.88.down_proj", "model.layers.4.mlp.experts.89.down_proj", "model.layers.4.mlp.experts.90.down_proj", "model.layers.4.mlp.experts.91.down_proj", "model.layers.4.mlp.experts.92.down_proj", "model.layers.4.mlp.experts.93.down_proj", "model.layers.4.mlp.experts.94.down_proj", "model.layers.4.mlp.experts.95.down_proj", "model.layers.4.mlp.experts.96.down_proj", "model.layers.4.mlp.experts.97.down_proj", "model.layers.4.mlp.experts.98.down_proj", "model.layers.4.mlp.experts.99.down_proj", "model.layers.4.mlp.experts.100.down_proj", "model.layers.4.mlp.experts.101.down_proj", "model.layers.4.mlp.experts.102.down_proj", "model.layers.4.mlp.experts.103.down_proj", "model.layers.4.mlp.experts.104.down_proj", "model.layers.4.mlp.experts.105.down_proj", "model.layers.4.mlp.experts.106.down_proj", "model.layers.4.mlp.experts.107.down_proj", "model.layers.4.mlp.experts.108.down_proj", "model.layers.4.mlp.experts.109.down_proj", "model.layers.4.mlp.experts.110.down_proj", "model.layers.4.mlp.experts.111.down_proj", "model.layers.4.mlp.experts.112.down_proj", "model.layers.4.mlp.experts.113.down_proj", "model.layers.4.mlp.experts.114.down_proj", "model.layers.4.mlp.experts.115.down_proj", "model.layers.4.mlp.experts.116.down_proj", "model.layers.4.mlp.experts.117.down_proj", "model.layers.4.mlp.experts.118.down_proj", "model.layers.4.mlp.experts.119.down_proj", "model.layers.4.mlp.experts.120.down_proj", "model.layers.4.mlp.experts.121.down_proj", "model.layers.4.mlp.experts.122.down_proj", "model.layers.4.mlp.experts.123.down_proj", "model.layers.4.mlp.experts.124.down_proj", "model.layers.4.mlp.experts.125.down_proj", "model.layers.4.mlp.experts.126.down_proj", "model.layers.4.mlp.experts.127.down_proj", "model.layers.4.mlp.experts.128.down_proj", "model.layers.4.mlp.experts.129.down_proj", "model.layers.4.mlp.experts.130.down_proj", "model.layers.4.mlp.experts.131.down_proj", "model.layers.4.mlp.experts.132.down_proj", "model.layers.4.mlp.experts.133.down_proj", "model.layers.4.mlp.experts.134.down_proj", "model.layers.4.mlp.experts.135.down_proj", "model.layers.4.mlp.experts.136.down_proj", "model.layers.4.mlp.experts.137.down_proj", "model.layers.4.mlp.experts.138.down_proj", "model.layers.4.mlp.experts.139.down_proj", "model.layers.4.mlp.experts.140.down_proj", "model.layers.4.mlp.experts.141.down_proj", "model.layers.4.mlp.experts.142.down_proj", "model.layers.4.mlp.experts.143.down_proj", "model.layers.4.mlp.experts.144.down_proj", "model.layers.4.mlp.experts.145.down_proj", "model.layers.4.mlp.experts.146.down_proj", "model.layers.4.mlp.experts.147.down_proj", "model.layers.4.mlp.experts.148.down_proj", "model.layers.4.mlp.experts.149.down_proj", "model.layers.4.mlp.experts.150.down_proj", "model.layers.4.mlp.experts.151.down_proj", "model.layers.4.mlp.experts.152.down_proj", "model.layers.4.mlp.experts.153.down_proj", "model.layers.4.mlp.experts.154.down_proj", "model.layers.4.mlp.experts.155.down_proj", "model.layers.4.mlp.experts.156.down_proj", "model.layers.4.mlp.experts.157.down_proj", "model.layers.4.mlp.experts.158.down_proj", "model.layers.4.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 9.417455294169399e-06, "dbits": 2516582400 } ] }, { "idx": 27, "layers": [ "model.layers.5.self_attn.q_proj" ], "candidates": [ { "dkld": -2.625576162245129e-05, "dbits": 125829120 } ] }, { "idx": 28, "layers": [ "model.layers.5.self_attn.k_proj", "model.layers.5.self_attn.v_proj" ], "candidates": [ { "dkld": -1.9271005294285693e-05, "dbits": 20971520 } ] }, { "idx": 29, "layers": [ "model.layers.5.self_attn.o_proj" ], "candidates": [ { "dkld": -1.164893328677865e-05, "dbits": 125829120 } ] }, { "idx": 30, "layers": [ "model.layers.5.mlp.shared_experts.gate_proj", "model.layers.5.mlp.shared_experts.up_proj", "model.layers.5.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -8.125035674311264e-06, "dbits": 47185920 } ] }, { "idx": 31, "layers": [ "model.layers.5.mlp.experts.0.gate_proj", "model.layers.5.mlp.experts.1.gate_proj", "model.layers.5.mlp.experts.2.gate_proj", "model.layers.5.mlp.experts.3.gate_proj", "model.layers.5.mlp.experts.4.gate_proj", "model.layers.5.mlp.experts.5.gate_proj", "model.layers.5.mlp.experts.6.gate_proj", "model.layers.5.mlp.experts.7.gate_proj", "model.layers.5.mlp.experts.8.gate_proj", "model.layers.5.mlp.experts.9.gate_proj", "model.layers.5.mlp.experts.10.gate_proj", "model.layers.5.mlp.experts.11.gate_proj", "model.layers.5.mlp.experts.12.gate_proj", "model.layers.5.mlp.experts.13.gate_proj", "model.layers.5.mlp.experts.14.gate_proj", "model.layers.5.mlp.experts.15.gate_proj", "model.layers.5.mlp.experts.16.gate_proj", "model.layers.5.mlp.experts.17.gate_proj", "model.layers.5.mlp.experts.18.gate_proj", "model.layers.5.mlp.experts.19.gate_proj", "model.layers.5.mlp.experts.20.gate_proj", "model.layers.5.mlp.experts.21.gate_proj", "model.layers.5.mlp.experts.22.gate_proj", "model.layers.5.mlp.experts.23.gate_proj", "model.layers.5.mlp.experts.24.gate_proj", "model.layers.5.mlp.experts.25.gate_proj", "model.layers.5.mlp.experts.26.gate_proj", "model.layers.5.mlp.experts.27.gate_proj", "model.layers.5.mlp.experts.28.gate_proj", "model.layers.5.mlp.experts.29.gate_proj", "model.layers.5.mlp.experts.30.gate_proj", "model.layers.5.mlp.experts.31.gate_proj", "model.layers.5.mlp.experts.32.gate_proj", "model.layers.5.mlp.experts.33.gate_proj", "model.layers.5.mlp.experts.34.gate_proj", "model.layers.5.mlp.experts.35.gate_proj", "model.layers.5.mlp.experts.36.gate_proj", "model.layers.5.mlp.experts.37.gate_proj", "model.layers.5.mlp.experts.38.gate_proj", "model.layers.5.mlp.experts.39.gate_proj", "model.layers.5.mlp.experts.40.gate_proj", "model.layers.5.mlp.experts.41.gate_proj", "model.layers.5.mlp.experts.42.gate_proj", "model.layers.5.mlp.experts.43.gate_proj", "model.layers.5.mlp.experts.44.gate_proj", "model.layers.5.mlp.experts.45.gate_proj", "model.layers.5.mlp.experts.46.gate_proj", "model.layers.5.mlp.experts.47.gate_proj", "model.layers.5.mlp.experts.48.gate_proj", "model.layers.5.mlp.experts.49.gate_proj", "model.layers.5.mlp.experts.50.gate_proj", "model.layers.5.mlp.experts.51.gate_proj", "model.layers.5.mlp.experts.52.gate_proj", "model.layers.5.mlp.experts.53.gate_proj", "model.layers.5.mlp.experts.54.gate_proj", "model.layers.5.mlp.experts.55.gate_proj", "model.layers.5.mlp.experts.56.gate_proj", "model.layers.5.mlp.experts.57.gate_proj", "model.layers.5.mlp.experts.58.gate_proj", "model.layers.5.mlp.experts.59.gate_proj", "model.layers.5.mlp.experts.60.gate_proj", "model.layers.5.mlp.experts.61.gate_proj", "model.layers.5.mlp.experts.62.gate_proj", "model.layers.5.mlp.experts.63.gate_proj", "model.layers.5.mlp.experts.64.gate_proj", "model.layers.5.mlp.experts.65.gate_proj", "model.layers.5.mlp.experts.66.gate_proj", "model.layers.5.mlp.experts.67.gate_proj", "model.layers.5.mlp.experts.68.gate_proj", "model.layers.5.mlp.experts.69.gate_proj", "model.layers.5.mlp.experts.70.gate_proj", "model.layers.5.mlp.experts.71.gate_proj", "model.layers.5.mlp.experts.72.gate_proj", "model.layers.5.mlp.experts.73.gate_proj", "model.layers.5.mlp.experts.74.gate_proj", "model.layers.5.mlp.experts.75.gate_proj", "model.layers.5.mlp.experts.76.gate_proj", "model.layers.5.mlp.experts.77.gate_proj", "model.layers.5.mlp.experts.78.gate_proj", "model.layers.5.mlp.experts.79.gate_proj", "model.layers.5.mlp.experts.80.gate_proj", "model.layers.5.mlp.experts.81.gate_proj", "model.layers.5.mlp.experts.82.gate_proj", "model.layers.5.mlp.experts.83.gate_proj", "model.layers.5.mlp.experts.84.gate_proj", "model.layers.5.mlp.experts.85.gate_proj", "model.layers.5.mlp.experts.86.gate_proj", "model.layers.5.mlp.experts.87.gate_proj", "model.layers.5.mlp.experts.88.gate_proj", "model.layers.5.mlp.experts.89.gate_proj", "model.layers.5.mlp.experts.90.gate_proj", "model.layers.5.mlp.experts.91.gate_proj", "model.layers.5.mlp.experts.92.gate_proj", "model.layers.5.mlp.experts.93.gate_proj", "model.layers.5.mlp.experts.94.gate_proj", "model.layers.5.mlp.experts.95.gate_proj", "model.layers.5.mlp.experts.96.gate_proj", "model.layers.5.mlp.experts.97.gate_proj", "model.layers.5.mlp.experts.98.gate_proj", "model.layers.5.mlp.experts.99.gate_proj", "model.layers.5.mlp.experts.100.gate_proj", "model.layers.5.mlp.experts.101.gate_proj", "model.layers.5.mlp.experts.102.gate_proj", "model.layers.5.mlp.experts.103.gate_proj", "model.layers.5.mlp.experts.104.gate_proj", "model.layers.5.mlp.experts.105.gate_proj", "model.layers.5.mlp.experts.106.gate_proj", "model.layers.5.mlp.experts.107.gate_proj", "model.layers.5.mlp.experts.108.gate_proj", "model.layers.5.mlp.experts.109.gate_proj", "model.layers.5.mlp.experts.110.gate_proj", "model.layers.5.mlp.experts.111.gate_proj", "model.layers.5.mlp.experts.112.gate_proj", "model.layers.5.mlp.experts.113.gate_proj", "model.layers.5.mlp.experts.114.gate_proj", "model.layers.5.mlp.experts.115.gate_proj", "model.layers.5.mlp.experts.116.gate_proj", "model.layers.5.mlp.experts.117.gate_proj", "model.layers.5.mlp.experts.118.gate_proj", "model.layers.5.mlp.experts.119.gate_proj", "model.layers.5.mlp.experts.120.gate_proj", "model.layers.5.mlp.experts.121.gate_proj", "model.layers.5.mlp.experts.122.gate_proj", "model.layers.5.mlp.experts.123.gate_proj", "model.layers.5.mlp.experts.124.gate_proj", "model.layers.5.mlp.experts.125.gate_proj", "model.layers.5.mlp.experts.126.gate_proj", "model.layers.5.mlp.experts.127.gate_proj", "model.layers.5.mlp.experts.128.gate_proj", "model.layers.5.mlp.experts.129.gate_proj", "model.layers.5.mlp.experts.130.gate_proj", "model.layers.5.mlp.experts.131.gate_proj", "model.layers.5.mlp.experts.132.gate_proj", "model.layers.5.mlp.experts.133.gate_proj", "model.layers.5.mlp.experts.134.gate_proj", "model.layers.5.mlp.experts.135.gate_proj", "model.layers.5.mlp.experts.136.gate_proj", "model.layers.5.mlp.experts.137.gate_proj", "model.layers.5.mlp.experts.138.gate_proj", "model.layers.5.mlp.experts.139.gate_proj", "model.layers.5.mlp.experts.140.gate_proj", "model.layers.5.mlp.experts.141.gate_proj", "model.layers.5.mlp.experts.142.gate_proj", "model.layers.5.mlp.experts.143.gate_proj", "model.layers.5.mlp.experts.144.gate_proj", "model.layers.5.mlp.experts.145.gate_proj", "model.layers.5.mlp.experts.146.gate_proj", "model.layers.5.mlp.experts.147.gate_proj", "model.layers.5.mlp.experts.148.gate_proj", "model.layers.5.mlp.experts.149.gate_proj", "model.layers.5.mlp.experts.150.gate_proj", "model.layers.5.mlp.experts.151.gate_proj", "model.layers.5.mlp.experts.152.gate_proj", "model.layers.5.mlp.experts.153.gate_proj", "model.layers.5.mlp.experts.154.gate_proj", "model.layers.5.mlp.experts.155.gate_proj", "model.layers.5.mlp.experts.156.gate_proj", "model.layers.5.mlp.experts.157.gate_proj", "model.layers.5.mlp.experts.158.gate_proj", "model.layers.5.mlp.experts.159.gate_proj", "model.layers.5.mlp.experts.0.up_proj", "model.layers.5.mlp.experts.1.up_proj", "model.layers.5.mlp.experts.2.up_proj", "model.layers.5.mlp.experts.3.up_proj", "model.layers.5.mlp.experts.4.up_proj", "model.layers.5.mlp.experts.5.up_proj", "model.layers.5.mlp.experts.6.up_proj", "model.layers.5.mlp.experts.7.up_proj", "model.layers.5.mlp.experts.8.up_proj", "model.layers.5.mlp.experts.9.up_proj", "model.layers.5.mlp.experts.10.up_proj", "model.layers.5.mlp.experts.11.up_proj", "model.layers.5.mlp.experts.12.up_proj", "model.layers.5.mlp.experts.13.up_proj", "model.layers.5.mlp.experts.14.up_proj", "model.layers.5.mlp.experts.15.up_proj", "model.layers.5.mlp.experts.16.up_proj", "model.layers.5.mlp.experts.17.up_proj", "model.layers.5.mlp.experts.18.up_proj", "model.layers.5.mlp.experts.19.up_proj", "model.layers.5.mlp.experts.20.up_proj", "model.layers.5.mlp.experts.21.up_proj", "model.layers.5.mlp.experts.22.up_proj", "model.layers.5.mlp.experts.23.up_proj", "model.layers.5.mlp.experts.24.up_proj", "model.layers.5.mlp.experts.25.up_proj", "model.layers.5.mlp.experts.26.up_proj", "model.layers.5.mlp.experts.27.up_proj", "model.layers.5.mlp.experts.28.up_proj", "model.layers.5.mlp.experts.29.up_proj", "model.layers.5.mlp.experts.30.up_proj", "model.layers.5.mlp.experts.31.up_proj", "model.layers.5.mlp.experts.32.up_proj", "model.layers.5.mlp.experts.33.up_proj", "model.layers.5.mlp.experts.34.up_proj", "model.layers.5.mlp.experts.35.up_proj", "model.layers.5.mlp.experts.36.up_proj", "model.layers.5.mlp.experts.37.up_proj", "model.layers.5.mlp.experts.38.up_proj", "model.layers.5.mlp.experts.39.up_proj", "model.layers.5.mlp.experts.40.up_proj", "model.layers.5.mlp.experts.41.up_proj", "model.layers.5.mlp.experts.42.up_proj", "model.layers.5.mlp.experts.43.up_proj", "model.layers.5.mlp.experts.44.up_proj", "model.layers.5.mlp.experts.45.up_proj", "model.layers.5.mlp.experts.46.up_proj", "model.layers.5.mlp.experts.47.up_proj", "model.layers.5.mlp.experts.48.up_proj", "model.layers.5.mlp.experts.49.up_proj", "model.layers.5.mlp.experts.50.up_proj", "model.layers.5.mlp.experts.51.up_proj", "model.layers.5.mlp.experts.52.up_proj", "model.layers.5.mlp.experts.53.up_proj", "model.layers.5.mlp.experts.54.up_proj", "model.layers.5.mlp.experts.55.up_proj", "model.layers.5.mlp.experts.56.up_proj", "model.layers.5.mlp.experts.57.up_proj", "model.layers.5.mlp.experts.58.up_proj", "model.layers.5.mlp.experts.59.up_proj", "model.layers.5.mlp.experts.60.up_proj", "model.layers.5.mlp.experts.61.up_proj", "model.layers.5.mlp.experts.62.up_proj", "model.layers.5.mlp.experts.63.up_proj", "model.layers.5.mlp.experts.64.up_proj", "model.layers.5.mlp.experts.65.up_proj", "model.layers.5.mlp.experts.66.up_proj", "model.layers.5.mlp.experts.67.up_proj", "model.layers.5.mlp.experts.68.up_proj", "model.layers.5.mlp.experts.69.up_proj", "model.layers.5.mlp.experts.70.up_proj", "model.layers.5.mlp.experts.71.up_proj", "model.layers.5.mlp.experts.72.up_proj", "model.layers.5.mlp.experts.73.up_proj", "model.layers.5.mlp.experts.74.up_proj", "model.layers.5.mlp.experts.75.up_proj", "model.layers.5.mlp.experts.76.up_proj", "model.layers.5.mlp.experts.77.up_proj", "model.layers.5.mlp.experts.78.up_proj", "model.layers.5.mlp.experts.79.up_proj", "model.layers.5.mlp.experts.80.up_proj", "model.layers.5.mlp.experts.81.up_proj", "model.layers.5.mlp.experts.82.up_proj", "model.layers.5.mlp.experts.83.up_proj", "model.layers.5.mlp.experts.84.up_proj", "model.layers.5.mlp.experts.85.up_proj", "model.layers.5.mlp.experts.86.up_proj", "model.layers.5.mlp.experts.87.up_proj", "model.layers.5.mlp.experts.88.up_proj", "model.layers.5.mlp.experts.89.up_proj", "model.layers.5.mlp.experts.90.up_proj", "model.layers.5.mlp.experts.91.up_proj", "model.layers.5.mlp.experts.92.up_proj", "model.layers.5.mlp.experts.93.up_proj", "model.layers.5.mlp.experts.94.up_proj", "model.layers.5.mlp.experts.95.up_proj", "model.layers.5.mlp.experts.96.up_proj", "model.layers.5.mlp.experts.97.up_proj", "model.layers.5.mlp.experts.98.up_proj", "model.layers.5.mlp.experts.99.up_proj", "model.layers.5.mlp.experts.100.up_proj", "model.layers.5.mlp.experts.101.up_proj", "model.layers.5.mlp.experts.102.up_proj", "model.layers.5.mlp.experts.103.up_proj", "model.layers.5.mlp.experts.104.up_proj", "model.layers.5.mlp.experts.105.up_proj", "model.layers.5.mlp.experts.106.up_proj", "model.layers.5.mlp.experts.107.up_proj", "model.layers.5.mlp.experts.108.up_proj", "model.layers.5.mlp.experts.109.up_proj", "model.layers.5.mlp.experts.110.up_proj", "model.layers.5.mlp.experts.111.up_proj", "model.layers.5.mlp.experts.112.up_proj", "model.layers.5.mlp.experts.113.up_proj", "model.layers.5.mlp.experts.114.up_proj", "model.layers.5.mlp.experts.115.up_proj", "model.layers.5.mlp.experts.116.up_proj", "model.layers.5.mlp.experts.117.up_proj", "model.layers.5.mlp.experts.118.up_proj", "model.layers.5.mlp.experts.119.up_proj", "model.layers.5.mlp.experts.120.up_proj", "model.layers.5.mlp.experts.121.up_proj", "model.layers.5.mlp.experts.122.up_proj", "model.layers.5.mlp.experts.123.up_proj", "model.layers.5.mlp.experts.124.up_proj", "model.layers.5.mlp.experts.125.up_proj", "model.layers.5.mlp.experts.126.up_proj", "model.layers.5.mlp.experts.127.up_proj", "model.layers.5.mlp.experts.128.up_proj", "model.layers.5.mlp.experts.129.up_proj", "model.layers.5.mlp.experts.130.up_proj", "model.layers.5.mlp.experts.131.up_proj", "model.layers.5.mlp.experts.132.up_proj", "model.layers.5.mlp.experts.133.up_proj", "model.layers.5.mlp.experts.134.up_proj", "model.layers.5.mlp.experts.135.up_proj", "model.layers.5.mlp.experts.136.up_proj", "model.layers.5.mlp.experts.137.up_proj", "model.layers.5.mlp.experts.138.up_proj", "model.layers.5.mlp.experts.139.up_proj", "model.layers.5.mlp.experts.140.up_proj", "model.layers.5.mlp.experts.141.up_proj", "model.layers.5.mlp.experts.142.up_proj", "model.layers.5.mlp.experts.143.up_proj", "model.layers.5.mlp.experts.144.up_proj", "model.layers.5.mlp.experts.145.up_proj", "model.layers.5.mlp.experts.146.up_proj", "model.layers.5.mlp.experts.147.up_proj", "model.layers.5.mlp.experts.148.up_proj", "model.layers.5.mlp.experts.149.up_proj", "model.layers.5.mlp.experts.150.up_proj", "model.layers.5.mlp.experts.151.up_proj", "model.layers.5.mlp.experts.152.up_proj", "model.layers.5.mlp.experts.153.up_proj", "model.layers.5.mlp.experts.154.up_proj", "model.layers.5.mlp.experts.155.up_proj", "model.layers.5.mlp.experts.156.up_proj", "model.layers.5.mlp.experts.157.up_proj", "model.layers.5.mlp.experts.158.up_proj", "model.layers.5.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -4.586299182847138e-05, "dbits": 5033164800 } ] }, { "idx": 32, "layers": [ "model.layers.5.mlp.experts.0.down_proj", "model.layers.5.mlp.experts.1.down_proj", "model.layers.5.mlp.experts.2.down_proj", "model.layers.5.mlp.experts.3.down_proj", "model.layers.5.mlp.experts.4.down_proj", "model.layers.5.mlp.experts.5.down_proj", "model.layers.5.mlp.experts.6.down_proj", "model.layers.5.mlp.experts.7.down_proj", "model.layers.5.mlp.experts.8.down_proj", "model.layers.5.mlp.experts.9.down_proj", "model.layers.5.mlp.experts.10.down_proj", "model.layers.5.mlp.experts.11.down_proj", "model.layers.5.mlp.experts.12.down_proj", "model.layers.5.mlp.experts.13.down_proj", "model.layers.5.mlp.experts.14.down_proj", "model.layers.5.mlp.experts.15.down_proj", "model.layers.5.mlp.experts.16.down_proj", "model.layers.5.mlp.experts.17.down_proj", "model.layers.5.mlp.experts.18.down_proj", "model.layers.5.mlp.experts.19.down_proj", "model.layers.5.mlp.experts.20.down_proj", "model.layers.5.mlp.experts.21.down_proj", "model.layers.5.mlp.experts.22.down_proj", "model.layers.5.mlp.experts.23.down_proj", "model.layers.5.mlp.experts.24.down_proj", "model.layers.5.mlp.experts.25.down_proj", "model.layers.5.mlp.experts.26.down_proj", "model.layers.5.mlp.experts.27.down_proj", "model.layers.5.mlp.experts.28.down_proj", "model.layers.5.mlp.experts.29.down_proj", "model.layers.5.mlp.experts.30.down_proj", "model.layers.5.mlp.experts.31.down_proj", "model.layers.5.mlp.experts.32.down_proj", "model.layers.5.mlp.experts.33.down_proj", "model.layers.5.mlp.experts.34.down_proj", "model.layers.5.mlp.experts.35.down_proj", "model.layers.5.mlp.experts.36.down_proj", "model.layers.5.mlp.experts.37.down_proj", "model.layers.5.mlp.experts.38.down_proj", "model.layers.5.mlp.experts.39.down_proj", "model.layers.5.mlp.experts.40.down_proj", "model.layers.5.mlp.experts.41.down_proj", "model.layers.5.mlp.experts.42.down_proj", "model.layers.5.mlp.experts.43.down_proj", "model.layers.5.mlp.experts.44.down_proj", "model.layers.5.mlp.experts.45.down_proj", "model.layers.5.mlp.experts.46.down_proj", "model.layers.5.mlp.experts.47.down_proj", "model.layers.5.mlp.experts.48.down_proj", "model.layers.5.mlp.experts.49.down_proj", "model.layers.5.mlp.experts.50.down_proj", "model.layers.5.mlp.experts.51.down_proj", "model.layers.5.mlp.experts.52.down_proj", "model.layers.5.mlp.experts.53.down_proj", "model.layers.5.mlp.experts.54.down_proj", "model.layers.5.mlp.experts.55.down_proj", "model.layers.5.mlp.experts.56.down_proj", "model.layers.5.mlp.experts.57.down_proj", "model.layers.5.mlp.experts.58.down_proj", "model.layers.5.mlp.experts.59.down_proj", "model.layers.5.mlp.experts.60.down_proj", "model.layers.5.mlp.experts.61.down_proj", "model.layers.5.mlp.experts.62.down_proj", "model.layers.5.mlp.experts.63.down_proj", "model.layers.5.mlp.experts.64.down_proj", "model.layers.5.mlp.experts.65.down_proj", "model.layers.5.mlp.experts.66.down_proj", "model.layers.5.mlp.experts.67.down_proj", "model.layers.5.mlp.experts.68.down_proj", "model.layers.5.mlp.experts.69.down_proj", "model.layers.5.mlp.experts.70.down_proj", "model.layers.5.mlp.experts.71.down_proj", "model.layers.5.mlp.experts.72.down_proj", "model.layers.5.mlp.experts.73.down_proj", "model.layers.5.mlp.experts.74.down_proj", "model.layers.5.mlp.experts.75.down_proj", "model.layers.5.mlp.experts.76.down_proj", "model.layers.5.mlp.experts.77.down_proj", "model.layers.5.mlp.experts.78.down_proj", "model.layers.5.mlp.experts.79.down_proj", "model.layers.5.mlp.experts.80.down_proj", "model.layers.5.mlp.experts.81.down_proj", "model.layers.5.mlp.experts.82.down_proj", "model.layers.5.mlp.experts.83.down_proj", "model.layers.5.mlp.experts.84.down_proj", "model.layers.5.mlp.experts.85.down_proj", "model.layers.5.mlp.experts.86.down_proj", "model.layers.5.mlp.experts.87.down_proj", "model.layers.5.mlp.experts.88.down_proj", "model.layers.5.mlp.experts.89.down_proj", "model.layers.5.mlp.experts.90.down_proj", "model.layers.5.mlp.experts.91.down_proj", "model.layers.5.mlp.experts.92.down_proj", "model.layers.5.mlp.experts.93.down_proj", "model.layers.5.mlp.experts.94.down_proj", "model.layers.5.mlp.experts.95.down_proj", "model.layers.5.mlp.experts.96.down_proj", "model.layers.5.mlp.experts.97.down_proj", "model.layers.5.mlp.experts.98.down_proj", "model.layers.5.mlp.experts.99.down_proj", "model.layers.5.mlp.experts.100.down_proj", "model.layers.5.mlp.experts.101.down_proj", "model.layers.5.mlp.experts.102.down_proj", "model.layers.5.mlp.experts.103.down_proj", "model.layers.5.mlp.experts.104.down_proj", "model.layers.5.mlp.experts.105.down_proj", "model.layers.5.mlp.experts.106.down_proj", "model.layers.5.mlp.experts.107.down_proj", "model.layers.5.mlp.experts.108.down_proj", "model.layers.5.mlp.experts.109.down_proj", "model.layers.5.mlp.experts.110.down_proj", "model.layers.5.mlp.experts.111.down_proj", "model.layers.5.mlp.experts.112.down_proj", "model.layers.5.mlp.experts.113.down_proj", "model.layers.5.mlp.experts.114.down_proj", "model.layers.5.mlp.experts.115.down_proj", "model.layers.5.mlp.experts.116.down_proj", "model.layers.5.mlp.experts.117.down_proj", "model.layers.5.mlp.experts.118.down_proj", "model.layers.5.mlp.experts.119.down_proj", "model.layers.5.mlp.experts.120.down_proj", "model.layers.5.mlp.experts.121.down_proj", "model.layers.5.mlp.experts.122.down_proj", "model.layers.5.mlp.experts.123.down_proj", "model.layers.5.mlp.experts.124.down_proj", "model.layers.5.mlp.experts.125.down_proj", "model.layers.5.mlp.experts.126.down_proj", "model.layers.5.mlp.experts.127.down_proj", "model.layers.5.mlp.experts.128.down_proj", "model.layers.5.mlp.experts.129.down_proj", "model.layers.5.mlp.experts.130.down_proj", "model.layers.5.mlp.experts.131.down_proj", "model.layers.5.mlp.experts.132.down_proj", "model.layers.5.mlp.experts.133.down_proj", "model.layers.5.mlp.experts.134.down_proj", "model.layers.5.mlp.experts.135.down_proj", "model.layers.5.mlp.experts.136.down_proj", "model.layers.5.mlp.experts.137.down_proj", "model.layers.5.mlp.experts.138.down_proj", "model.layers.5.mlp.experts.139.down_proj", "model.layers.5.mlp.experts.140.down_proj", "model.layers.5.mlp.experts.141.down_proj", "model.layers.5.mlp.experts.142.down_proj", "model.layers.5.mlp.experts.143.down_proj", "model.layers.5.mlp.experts.144.down_proj", "model.layers.5.mlp.experts.145.down_proj", "model.layers.5.mlp.experts.146.down_proj", "model.layers.5.mlp.experts.147.down_proj", "model.layers.5.mlp.experts.148.down_proj", "model.layers.5.mlp.experts.149.down_proj", "model.layers.5.mlp.experts.150.down_proj", "model.layers.5.mlp.experts.151.down_proj", "model.layers.5.mlp.experts.152.down_proj", "model.layers.5.mlp.experts.153.down_proj", "model.layers.5.mlp.experts.154.down_proj", "model.layers.5.mlp.experts.155.down_proj", "model.layers.5.mlp.experts.156.down_proj", "model.layers.5.mlp.experts.157.down_proj", "model.layers.5.mlp.experts.158.down_proj", "model.layers.5.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.171940530184665e-05, "dbits": 2516582400 } ] }, { "idx": 33, "layers": [ "model.layers.6.self_attn.q_proj" ], "candidates": [ { "dkld": -4.42416639998553e-06, "dbits": 125829120 } ] }, { "idx": 34, "layers": [ "model.layers.6.self_attn.k_proj", "model.layers.6.self_attn.v_proj" ], "candidates": [ { "dkld": -7.84349977038805e-06, "dbits": 20971520 } ] }, { "idx": 35, "layers": [ "model.layers.6.self_attn.o_proj" ], "candidates": [ { "dkld": -1.5143421478569074e-06, "dbits": 125829120 } ] }, { "idx": 36, "layers": [ "model.layers.6.mlp.shared_experts.gate_proj", "model.layers.6.mlp.shared_experts.up_proj", "model.layers.6.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -4.0299695683643005e-06, "dbits": 47185920 } ] }, { "idx": 37, "layers": [ "model.layers.6.mlp.experts.0.gate_proj", "model.layers.6.mlp.experts.1.gate_proj", "model.layers.6.mlp.experts.2.gate_proj", "model.layers.6.mlp.experts.3.gate_proj", "model.layers.6.mlp.experts.4.gate_proj", "model.layers.6.mlp.experts.5.gate_proj", "model.layers.6.mlp.experts.6.gate_proj", "model.layers.6.mlp.experts.7.gate_proj", "model.layers.6.mlp.experts.8.gate_proj", "model.layers.6.mlp.experts.9.gate_proj", "model.layers.6.mlp.experts.10.gate_proj", "model.layers.6.mlp.experts.11.gate_proj", "model.layers.6.mlp.experts.12.gate_proj", "model.layers.6.mlp.experts.13.gate_proj", "model.layers.6.mlp.experts.14.gate_proj", "model.layers.6.mlp.experts.15.gate_proj", "model.layers.6.mlp.experts.16.gate_proj", "model.layers.6.mlp.experts.17.gate_proj", "model.layers.6.mlp.experts.18.gate_proj", "model.layers.6.mlp.experts.19.gate_proj", "model.layers.6.mlp.experts.20.gate_proj", "model.layers.6.mlp.experts.21.gate_proj", "model.layers.6.mlp.experts.22.gate_proj", "model.layers.6.mlp.experts.23.gate_proj", "model.layers.6.mlp.experts.24.gate_proj", "model.layers.6.mlp.experts.25.gate_proj", "model.layers.6.mlp.experts.26.gate_proj", "model.layers.6.mlp.experts.27.gate_proj", "model.layers.6.mlp.experts.28.gate_proj", "model.layers.6.mlp.experts.29.gate_proj", "model.layers.6.mlp.experts.30.gate_proj", "model.layers.6.mlp.experts.31.gate_proj", "model.layers.6.mlp.experts.32.gate_proj", "model.layers.6.mlp.experts.33.gate_proj", "model.layers.6.mlp.experts.34.gate_proj", "model.layers.6.mlp.experts.35.gate_proj", "model.layers.6.mlp.experts.36.gate_proj", "model.layers.6.mlp.experts.37.gate_proj", "model.layers.6.mlp.experts.38.gate_proj", "model.layers.6.mlp.experts.39.gate_proj", "model.layers.6.mlp.experts.40.gate_proj", "model.layers.6.mlp.experts.41.gate_proj", "model.layers.6.mlp.experts.42.gate_proj", "model.layers.6.mlp.experts.43.gate_proj", "model.layers.6.mlp.experts.44.gate_proj", "model.layers.6.mlp.experts.45.gate_proj", "model.layers.6.mlp.experts.46.gate_proj", "model.layers.6.mlp.experts.47.gate_proj", "model.layers.6.mlp.experts.48.gate_proj", "model.layers.6.mlp.experts.49.gate_proj", "model.layers.6.mlp.experts.50.gate_proj", "model.layers.6.mlp.experts.51.gate_proj", "model.layers.6.mlp.experts.52.gate_proj", "model.layers.6.mlp.experts.53.gate_proj", "model.layers.6.mlp.experts.54.gate_proj", "model.layers.6.mlp.experts.55.gate_proj", "model.layers.6.mlp.experts.56.gate_proj", "model.layers.6.mlp.experts.57.gate_proj", "model.layers.6.mlp.experts.58.gate_proj", "model.layers.6.mlp.experts.59.gate_proj", "model.layers.6.mlp.experts.60.gate_proj", "model.layers.6.mlp.experts.61.gate_proj", "model.layers.6.mlp.experts.62.gate_proj", "model.layers.6.mlp.experts.63.gate_proj", "model.layers.6.mlp.experts.64.gate_proj", "model.layers.6.mlp.experts.65.gate_proj", "model.layers.6.mlp.experts.66.gate_proj", "model.layers.6.mlp.experts.67.gate_proj", "model.layers.6.mlp.experts.68.gate_proj", "model.layers.6.mlp.experts.69.gate_proj", "model.layers.6.mlp.experts.70.gate_proj", "model.layers.6.mlp.experts.71.gate_proj", "model.layers.6.mlp.experts.72.gate_proj", "model.layers.6.mlp.experts.73.gate_proj", "model.layers.6.mlp.experts.74.gate_proj", "model.layers.6.mlp.experts.75.gate_proj", "model.layers.6.mlp.experts.76.gate_proj", "model.layers.6.mlp.experts.77.gate_proj", "model.layers.6.mlp.experts.78.gate_proj", "model.layers.6.mlp.experts.79.gate_proj", "model.layers.6.mlp.experts.80.gate_proj", "model.layers.6.mlp.experts.81.gate_proj", "model.layers.6.mlp.experts.82.gate_proj", "model.layers.6.mlp.experts.83.gate_proj", "model.layers.6.mlp.experts.84.gate_proj", "model.layers.6.mlp.experts.85.gate_proj", "model.layers.6.mlp.experts.86.gate_proj", "model.layers.6.mlp.experts.87.gate_proj", "model.layers.6.mlp.experts.88.gate_proj", "model.layers.6.mlp.experts.89.gate_proj", "model.layers.6.mlp.experts.90.gate_proj", "model.layers.6.mlp.experts.91.gate_proj", "model.layers.6.mlp.experts.92.gate_proj", "model.layers.6.mlp.experts.93.gate_proj", "model.layers.6.mlp.experts.94.gate_proj", "model.layers.6.mlp.experts.95.gate_proj", "model.layers.6.mlp.experts.96.gate_proj", "model.layers.6.mlp.experts.97.gate_proj", "model.layers.6.mlp.experts.98.gate_proj", "model.layers.6.mlp.experts.99.gate_proj", "model.layers.6.mlp.experts.100.gate_proj", "model.layers.6.mlp.experts.101.gate_proj", "model.layers.6.mlp.experts.102.gate_proj", "model.layers.6.mlp.experts.103.gate_proj", "model.layers.6.mlp.experts.104.gate_proj", "model.layers.6.mlp.experts.105.gate_proj", "model.layers.6.mlp.experts.106.gate_proj", "model.layers.6.mlp.experts.107.gate_proj", "model.layers.6.mlp.experts.108.gate_proj", "model.layers.6.mlp.experts.109.gate_proj", "model.layers.6.mlp.experts.110.gate_proj", "model.layers.6.mlp.experts.111.gate_proj", "model.layers.6.mlp.experts.112.gate_proj", "model.layers.6.mlp.experts.113.gate_proj", "model.layers.6.mlp.experts.114.gate_proj", "model.layers.6.mlp.experts.115.gate_proj", "model.layers.6.mlp.experts.116.gate_proj", "model.layers.6.mlp.experts.117.gate_proj", "model.layers.6.mlp.experts.118.gate_proj", "model.layers.6.mlp.experts.119.gate_proj", "model.layers.6.mlp.experts.120.gate_proj", "model.layers.6.mlp.experts.121.gate_proj", "model.layers.6.mlp.experts.122.gate_proj", "model.layers.6.mlp.experts.123.gate_proj", "model.layers.6.mlp.experts.124.gate_proj", "model.layers.6.mlp.experts.125.gate_proj", "model.layers.6.mlp.experts.126.gate_proj", "model.layers.6.mlp.experts.127.gate_proj", "model.layers.6.mlp.experts.128.gate_proj", "model.layers.6.mlp.experts.129.gate_proj", "model.layers.6.mlp.experts.130.gate_proj", "model.layers.6.mlp.experts.131.gate_proj", "model.layers.6.mlp.experts.132.gate_proj", "model.layers.6.mlp.experts.133.gate_proj", "model.layers.6.mlp.experts.134.gate_proj", "model.layers.6.mlp.experts.135.gate_proj", "model.layers.6.mlp.experts.136.gate_proj", "model.layers.6.mlp.experts.137.gate_proj", "model.layers.6.mlp.experts.138.gate_proj", "model.layers.6.mlp.experts.139.gate_proj", "model.layers.6.mlp.experts.140.gate_proj", "model.layers.6.mlp.experts.141.gate_proj", "model.layers.6.mlp.experts.142.gate_proj", "model.layers.6.mlp.experts.143.gate_proj", "model.layers.6.mlp.experts.144.gate_proj", "model.layers.6.mlp.experts.145.gate_proj", "model.layers.6.mlp.experts.146.gate_proj", "model.layers.6.mlp.experts.147.gate_proj", "model.layers.6.mlp.experts.148.gate_proj", "model.layers.6.mlp.experts.149.gate_proj", "model.layers.6.mlp.experts.150.gate_proj", "model.layers.6.mlp.experts.151.gate_proj", "model.layers.6.mlp.experts.152.gate_proj", "model.layers.6.mlp.experts.153.gate_proj", "model.layers.6.mlp.experts.154.gate_proj", "model.layers.6.mlp.experts.155.gate_proj", "model.layers.6.mlp.experts.156.gate_proj", "model.layers.6.mlp.experts.157.gate_proj", "model.layers.6.mlp.experts.158.gate_proj", "model.layers.6.mlp.experts.159.gate_proj", "model.layers.6.mlp.experts.0.up_proj", "model.layers.6.mlp.experts.1.up_proj", "model.layers.6.mlp.experts.2.up_proj", "model.layers.6.mlp.experts.3.up_proj", "model.layers.6.mlp.experts.4.up_proj", "model.layers.6.mlp.experts.5.up_proj", "model.layers.6.mlp.experts.6.up_proj", "model.layers.6.mlp.experts.7.up_proj", "model.layers.6.mlp.experts.8.up_proj", "model.layers.6.mlp.experts.9.up_proj", "model.layers.6.mlp.experts.10.up_proj", "model.layers.6.mlp.experts.11.up_proj", "model.layers.6.mlp.experts.12.up_proj", "model.layers.6.mlp.experts.13.up_proj", "model.layers.6.mlp.experts.14.up_proj", "model.layers.6.mlp.experts.15.up_proj", "model.layers.6.mlp.experts.16.up_proj", "model.layers.6.mlp.experts.17.up_proj", "model.layers.6.mlp.experts.18.up_proj", "model.layers.6.mlp.experts.19.up_proj", "model.layers.6.mlp.experts.20.up_proj", "model.layers.6.mlp.experts.21.up_proj", "model.layers.6.mlp.experts.22.up_proj", "model.layers.6.mlp.experts.23.up_proj", "model.layers.6.mlp.experts.24.up_proj", "model.layers.6.mlp.experts.25.up_proj", "model.layers.6.mlp.experts.26.up_proj", "model.layers.6.mlp.experts.27.up_proj", "model.layers.6.mlp.experts.28.up_proj", "model.layers.6.mlp.experts.29.up_proj", "model.layers.6.mlp.experts.30.up_proj", "model.layers.6.mlp.experts.31.up_proj", "model.layers.6.mlp.experts.32.up_proj", "model.layers.6.mlp.experts.33.up_proj", "model.layers.6.mlp.experts.34.up_proj", "model.layers.6.mlp.experts.35.up_proj", "model.layers.6.mlp.experts.36.up_proj", "model.layers.6.mlp.experts.37.up_proj", "model.layers.6.mlp.experts.38.up_proj", "model.layers.6.mlp.experts.39.up_proj", "model.layers.6.mlp.experts.40.up_proj", "model.layers.6.mlp.experts.41.up_proj", "model.layers.6.mlp.experts.42.up_proj", "model.layers.6.mlp.experts.43.up_proj", "model.layers.6.mlp.experts.44.up_proj", "model.layers.6.mlp.experts.45.up_proj", "model.layers.6.mlp.experts.46.up_proj", "model.layers.6.mlp.experts.47.up_proj", "model.layers.6.mlp.experts.48.up_proj", "model.layers.6.mlp.experts.49.up_proj", "model.layers.6.mlp.experts.50.up_proj", "model.layers.6.mlp.experts.51.up_proj", "model.layers.6.mlp.experts.52.up_proj", "model.layers.6.mlp.experts.53.up_proj", "model.layers.6.mlp.experts.54.up_proj", "model.layers.6.mlp.experts.55.up_proj", "model.layers.6.mlp.experts.56.up_proj", "model.layers.6.mlp.experts.57.up_proj", "model.layers.6.mlp.experts.58.up_proj", "model.layers.6.mlp.experts.59.up_proj", "model.layers.6.mlp.experts.60.up_proj", "model.layers.6.mlp.experts.61.up_proj", "model.layers.6.mlp.experts.62.up_proj", "model.layers.6.mlp.experts.63.up_proj", "model.layers.6.mlp.experts.64.up_proj", "model.layers.6.mlp.experts.65.up_proj", "model.layers.6.mlp.experts.66.up_proj", "model.layers.6.mlp.experts.67.up_proj", "model.layers.6.mlp.experts.68.up_proj", "model.layers.6.mlp.experts.69.up_proj", "model.layers.6.mlp.experts.70.up_proj", "model.layers.6.mlp.experts.71.up_proj", "model.layers.6.mlp.experts.72.up_proj", "model.layers.6.mlp.experts.73.up_proj", "model.layers.6.mlp.experts.74.up_proj", "model.layers.6.mlp.experts.75.up_proj", "model.layers.6.mlp.experts.76.up_proj", "model.layers.6.mlp.experts.77.up_proj", "model.layers.6.mlp.experts.78.up_proj", "model.layers.6.mlp.experts.79.up_proj", "model.layers.6.mlp.experts.80.up_proj", "model.layers.6.mlp.experts.81.up_proj", "model.layers.6.mlp.experts.82.up_proj", "model.layers.6.mlp.experts.83.up_proj", "model.layers.6.mlp.experts.84.up_proj", "model.layers.6.mlp.experts.85.up_proj", "model.layers.6.mlp.experts.86.up_proj", "model.layers.6.mlp.experts.87.up_proj", "model.layers.6.mlp.experts.88.up_proj", "model.layers.6.mlp.experts.89.up_proj", "model.layers.6.mlp.experts.90.up_proj", "model.layers.6.mlp.experts.91.up_proj", "model.layers.6.mlp.experts.92.up_proj", "model.layers.6.mlp.experts.93.up_proj", "model.layers.6.mlp.experts.94.up_proj", "model.layers.6.mlp.experts.95.up_proj", "model.layers.6.mlp.experts.96.up_proj", "model.layers.6.mlp.experts.97.up_proj", "model.layers.6.mlp.experts.98.up_proj", "model.layers.6.mlp.experts.99.up_proj", "model.layers.6.mlp.experts.100.up_proj", "model.layers.6.mlp.experts.101.up_proj", "model.layers.6.mlp.experts.102.up_proj", "model.layers.6.mlp.experts.103.up_proj", "model.layers.6.mlp.experts.104.up_proj", "model.layers.6.mlp.experts.105.up_proj", "model.layers.6.mlp.experts.106.up_proj", "model.layers.6.mlp.experts.107.up_proj", "model.layers.6.mlp.experts.108.up_proj", "model.layers.6.mlp.experts.109.up_proj", "model.layers.6.mlp.experts.110.up_proj", "model.layers.6.mlp.experts.111.up_proj", "model.layers.6.mlp.experts.112.up_proj", "model.layers.6.mlp.experts.113.up_proj", "model.layers.6.mlp.experts.114.up_proj", "model.layers.6.mlp.experts.115.up_proj", "model.layers.6.mlp.experts.116.up_proj", "model.layers.6.mlp.experts.117.up_proj", "model.layers.6.mlp.experts.118.up_proj", "model.layers.6.mlp.experts.119.up_proj", "model.layers.6.mlp.experts.120.up_proj", "model.layers.6.mlp.experts.121.up_proj", "model.layers.6.mlp.experts.122.up_proj", "model.layers.6.mlp.experts.123.up_proj", "model.layers.6.mlp.experts.124.up_proj", "model.layers.6.mlp.experts.125.up_proj", "model.layers.6.mlp.experts.126.up_proj", "model.layers.6.mlp.experts.127.up_proj", "model.layers.6.mlp.experts.128.up_proj", "model.layers.6.mlp.experts.129.up_proj", "model.layers.6.mlp.experts.130.up_proj", "model.layers.6.mlp.experts.131.up_proj", "model.layers.6.mlp.experts.132.up_proj", "model.layers.6.mlp.experts.133.up_proj", "model.layers.6.mlp.experts.134.up_proj", "model.layers.6.mlp.experts.135.up_proj", "model.layers.6.mlp.experts.136.up_proj", "model.layers.6.mlp.experts.137.up_proj", "model.layers.6.mlp.experts.138.up_proj", "model.layers.6.mlp.experts.139.up_proj", "model.layers.6.mlp.experts.140.up_proj", "model.layers.6.mlp.experts.141.up_proj", "model.layers.6.mlp.experts.142.up_proj", "model.layers.6.mlp.experts.143.up_proj", "model.layers.6.mlp.experts.144.up_proj", "model.layers.6.mlp.experts.145.up_proj", "model.layers.6.mlp.experts.146.up_proj", "model.layers.6.mlp.experts.147.up_proj", "model.layers.6.mlp.experts.148.up_proj", "model.layers.6.mlp.experts.149.up_proj", "model.layers.6.mlp.experts.150.up_proj", "model.layers.6.mlp.experts.151.up_proj", "model.layers.6.mlp.experts.152.up_proj", "model.layers.6.mlp.experts.153.up_proj", "model.layers.6.mlp.experts.154.up_proj", "model.layers.6.mlp.experts.155.up_proj", "model.layers.6.mlp.experts.156.up_proj", "model.layers.6.mlp.experts.157.up_proj", "model.layers.6.mlp.experts.158.up_proj", "model.layers.6.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.9010002617724283e-05, "dbits": 5033164800 } ] }, { "idx": 38, "layers": [ "model.layers.6.mlp.experts.0.down_proj", "model.layers.6.mlp.experts.1.down_proj", "model.layers.6.mlp.experts.2.down_proj", "model.layers.6.mlp.experts.3.down_proj", "model.layers.6.mlp.experts.4.down_proj", "model.layers.6.mlp.experts.5.down_proj", "model.layers.6.mlp.experts.6.down_proj", "model.layers.6.mlp.experts.7.down_proj", "model.layers.6.mlp.experts.8.down_proj", "model.layers.6.mlp.experts.9.down_proj", "model.layers.6.mlp.experts.10.down_proj", "model.layers.6.mlp.experts.11.down_proj", "model.layers.6.mlp.experts.12.down_proj", "model.layers.6.mlp.experts.13.down_proj", "model.layers.6.mlp.experts.14.down_proj", "model.layers.6.mlp.experts.15.down_proj", "model.layers.6.mlp.experts.16.down_proj", "model.layers.6.mlp.experts.17.down_proj", "model.layers.6.mlp.experts.18.down_proj", "model.layers.6.mlp.experts.19.down_proj", "model.layers.6.mlp.experts.20.down_proj", "model.layers.6.mlp.experts.21.down_proj", "model.layers.6.mlp.experts.22.down_proj", "model.layers.6.mlp.experts.23.down_proj", "model.layers.6.mlp.experts.24.down_proj", "model.layers.6.mlp.experts.25.down_proj", "model.layers.6.mlp.experts.26.down_proj", "model.layers.6.mlp.experts.27.down_proj", "model.layers.6.mlp.experts.28.down_proj", "model.layers.6.mlp.experts.29.down_proj", "model.layers.6.mlp.experts.30.down_proj", "model.layers.6.mlp.experts.31.down_proj", "model.layers.6.mlp.experts.32.down_proj", "model.layers.6.mlp.experts.33.down_proj", "model.layers.6.mlp.experts.34.down_proj", "model.layers.6.mlp.experts.35.down_proj", "model.layers.6.mlp.experts.36.down_proj", "model.layers.6.mlp.experts.37.down_proj", "model.layers.6.mlp.experts.38.down_proj", "model.layers.6.mlp.experts.39.down_proj", "model.layers.6.mlp.experts.40.down_proj", "model.layers.6.mlp.experts.41.down_proj", "model.layers.6.mlp.experts.42.down_proj", "model.layers.6.mlp.experts.43.down_proj", "model.layers.6.mlp.experts.44.down_proj", "model.layers.6.mlp.experts.45.down_proj", "model.layers.6.mlp.experts.46.down_proj", "model.layers.6.mlp.experts.47.down_proj", "model.layers.6.mlp.experts.48.down_proj", "model.layers.6.mlp.experts.49.down_proj", "model.layers.6.mlp.experts.50.down_proj", "model.layers.6.mlp.experts.51.down_proj", "model.layers.6.mlp.experts.52.down_proj", "model.layers.6.mlp.experts.53.down_proj", "model.layers.6.mlp.experts.54.down_proj", "model.layers.6.mlp.experts.55.down_proj", "model.layers.6.mlp.experts.56.down_proj", "model.layers.6.mlp.experts.57.down_proj", "model.layers.6.mlp.experts.58.down_proj", "model.layers.6.mlp.experts.59.down_proj", "model.layers.6.mlp.experts.60.down_proj", "model.layers.6.mlp.experts.61.down_proj", "model.layers.6.mlp.experts.62.down_proj", "model.layers.6.mlp.experts.63.down_proj", "model.layers.6.mlp.experts.64.down_proj", "model.layers.6.mlp.experts.65.down_proj", "model.layers.6.mlp.experts.66.down_proj", "model.layers.6.mlp.experts.67.down_proj", "model.layers.6.mlp.experts.68.down_proj", "model.layers.6.mlp.experts.69.down_proj", "model.layers.6.mlp.experts.70.down_proj", "model.layers.6.mlp.experts.71.down_proj", "model.layers.6.mlp.experts.72.down_proj", "model.layers.6.mlp.experts.73.down_proj", "model.layers.6.mlp.experts.74.down_proj", "model.layers.6.mlp.experts.75.down_proj", "model.layers.6.mlp.experts.76.down_proj", "model.layers.6.mlp.experts.77.down_proj", "model.layers.6.mlp.experts.78.down_proj", "model.layers.6.mlp.experts.79.down_proj", "model.layers.6.mlp.experts.80.down_proj", "model.layers.6.mlp.experts.81.down_proj", "model.layers.6.mlp.experts.82.down_proj", "model.layers.6.mlp.experts.83.down_proj", "model.layers.6.mlp.experts.84.down_proj", "model.layers.6.mlp.experts.85.down_proj", "model.layers.6.mlp.experts.86.down_proj", "model.layers.6.mlp.experts.87.down_proj", "model.layers.6.mlp.experts.88.down_proj", "model.layers.6.mlp.experts.89.down_proj", "model.layers.6.mlp.experts.90.down_proj", "model.layers.6.mlp.experts.91.down_proj", "model.layers.6.mlp.experts.92.down_proj", "model.layers.6.mlp.experts.93.down_proj", "model.layers.6.mlp.experts.94.down_proj", "model.layers.6.mlp.experts.95.down_proj", "model.layers.6.mlp.experts.96.down_proj", "model.layers.6.mlp.experts.97.down_proj", "model.layers.6.mlp.experts.98.down_proj", "model.layers.6.mlp.experts.99.down_proj", "model.layers.6.mlp.experts.100.down_proj", "model.layers.6.mlp.experts.101.down_proj", "model.layers.6.mlp.experts.102.down_proj", "model.layers.6.mlp.experts.103.down_proj", "model.layers.6.mlp.experts.104.down_proj", "model.layers.6.mlp.experts.105.down_proj", "model.layers.6.mlp.experts.106.down_proj", "model.layers.6.mlp.experts.107.down_proj", "model.layers.6.mlp.experts.108.down_proj", "model.layers.6.mlp.experts.109.down_proj", "model.layers.6.mlp.experts.110.down_proj", "model.layers.6.mlp.experts.111.down_proj", "model.layers.6.mlp.experts.112.down_proj", "model.layers.6.mlp.experts.113.down_proj", "model.layers.6.mlp.experts.114.down_proj", "model.layers.6.mlp.experts.115.down_proj", "model.layers.6.mlp.experts.116.down_proj", "model.layers.6.mlp.experts.117.down_proj", "model.layers.6.mlp.experts.118.down_proj", "model.layers.6.mlp.experts.119.down_proj", "model.layers.6.mlp.experts.120.down_proj", "model.layers.6.mlp.experts.121.down_proj", "model.layers.6.mlp.experts.122.down_proj", "model.layers.6.mlp.experts.123.down_proj", "model.layers.6.mlp.experts.124.down_proj", "model.layers.6.mlp.experts.125.down_proj", "model.layers.6.mlp.experts.126.down_proj", "model.layers.6.mlp.experts.127.down_proj", "model.layers.6.mlp.experts.128.down_proj", "model.layers.6.mlp.experts.129.down_proj", "model.layers.6.mlp.experts.130.down_proj", "model.layers.6.mlp.experts.131.down_proj", "model.layers.6.mlp.experts.132.down_proj", "model.layers.6.mlp.experts.133.down_proj", "model.layers.6.mlp.experts.134.down_proj", "model.layers.6.mlp.experts.135.down_proj", "model.layers.6.mlp.experts.136.down_proj", "model.layers.6.mlp.experts.137.down_proj", "model.layers.6.mlp.experts.138.down_proj", "model.layers.6.mlp.experts.139.down_proj", "model.layers.6.mlp.experts.140.down_proj", "model.layers.6.mlp.experts.141.down_proj", "model.layers.6.mlp.experts.142.down_proj", "model.layers.6.mlp.experts.143.down_proj", "model.layers.6.mlp.experts.144.down_proj", "model.layers.6.mlp.experts.145.down_proj", "model.layers.6.mlp.experts.146.down_proj", "model.layers.6.mlp.experts.147.down_proj", "model.layers.6.mlp.experts.148.down_proj", "model.layers.6.mlp.experts.149.down_proj", "model.layers.6.mlp.experts.150.down_proj", "model.layers.6.mlp.experts.151.down_proj", "model.layers.6.mlp.experts.152.down_proj", "model.layers.6.mlp.experts.153.down_proj", "model.layers.6.mlp.experts.154.down_proj", "model.layers.6.mlp.experts.155.down_proj", "model.layers.6.mlp.experts.156.down_proj", "model.layers.6.mlp.experts.157.down_proj", "model.layers.6.mlp.experts.158.down_proj", "model.layers.6.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.2100564567371986e-06, "dbits": 2516582400 } ] }, { "idx": 39, "layers": [ "model.layers.7.self_attn.q_proj" ], "candidates": [ { "dkld": -3.6452367203310154e-05, "dbits": 125829120 } ] }, { "idx": 40, "layers": [ "model.layers.7.self_attn.k_proj", "model.layers.7.self_attn.v_proj" ], "candidates": [ { "dkld": -1.768882211763414e-05, "dbits": 20971520 } ] }, { "idx": 41, "layers": [ "model.layers.7.self_attn.o_proj" ], "candidates": [ { "dkld": -4.9741147086024306e-05, "dbits": 125829120 } ] }, { "idx": 42, "layers": [ "model.layers.7.mlp.shared_experts.gate_proj", "model.layers.7.mlp.shared_experts.up_proj", "model.layers.7.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.934701042249803e-05, "dbits": 47185920 } ] }, { "idx": 43, "layers": [ "model.layers.7.mlp.experts.0.gate_proj", "model.layers.7.mlp.experts.1.gate_proj", "model.layers.7.mlp.experts.2.gate_proj", "model.layers.7.mlp.experts.3.gate_proj", "model.layers.7.mlp.experts.4.gate_proj", "model.layers.7.mlp.experts.5.gate_proj", "model.layers.7.mlp.experts.6.gate_proj", "model.layers.7.mlp.experts.7.gate_proj", "model.layers.7.mlp.experts.8.gate_proj", "model.layers.7.mlp.experts.9.gate_proj", "model.layers.7.mlp.experts.10.gate_proj", "model.layers.7.mlp.experts.11.gate_proj", "model.layers.7.mlp.experts.12.gate_proj", "model.layers.7.mlp.experts.13.gate_proj", "model.layers.7.mlp.experts.14.gate_proj", "model.layers.7.mlp.experts.15.gate_proj", "model.layers.7.mlp.experts.16.gate_proj", "model.layers.7.mlp.experts.17.gate_proj", "model.layers.7.mlp.experts.18.gate_proj", "model.layers.7.mlp.experts.19.gate_proj", "model.layers.7.mlp.experts.20.gate_proj", "model.layers.7.mlp.experts.21.gate_proj", "model.layers.7.mlp.experts.22.gate_proj", "model.layers.7.mlp.experts.23.gate_proj", "model.layers.7.mlp.experts.24.gate_proj", "model.layers.7.mlp.experts.25.gate_proj", "model.layers.7.mlp.experts.26.gate_proj", "model.layers.7.mlp.experts.27.gate_proj", "model.layers.7.mlp.experts.28.gate_proj", "model.layers.7.mlp.experts.29.gate_proj", "model.layers.7.mlp.experts.30.gate_proj", "model.layers.7.mlp.experts.31.gate_proj", "model.layers.7.mlp.experts.32.gate_proj", "model.layers.7.mlp.experts.33.gate_proj", "model.layers.7.mlp.experts.34.gate_proj", "model.layers.7.mlp.experts.35.gate_proj", "model.layers.7.mlp.experts.36.gate_proj", "model.layers.7.mlp.experts.37.gate_proj", "model.layers.7.mlp.experts.38.gate_proj", "model.layers.7.mlp.experts.39.gate_proj", "model.layers.7.mlp.experts.40.gate_proj", "model.layers.7.mlp.experts.41.gate_proj", "model.layers.7.mlp.experts.42.gate_proj", "model.layers.7.mlp.experts.43.gate_proj", "model.layers.7.mlp.experts.44.gate_proj", "model.layers.7.mlp.experts.45.gate_proj", "model.layers.7.mlp.experts.46.gate_proj", "model.layers.7.mlp.experts.47.gate_proj", "model.layers.7.mlp.experts.48.gate_proj", "model.layers.7.mlp.experts.49.gate_proj", "model.layers.7.mlp.experts.50.gate_proj", "model.layers.7.mlp.experts.51.gate_proj", "model.layers.7.mlp.experts.52.gate_proj", "model.layers.7.mlp.experts.53.gate_proj", "model.layers.7.mlp.experts.54.gate_proj", "model.layers.7.mlp.experts.55.gate_proj", "model.layers.7.mlp.experts.56.gate_proj", "model.layers.7.mlp.experts.57.gate_proj", "model.layers.7.mlp.experts.58.gate_proj", "model.layers.7.mlp.experts.59.gate_proj", "model.layers.7.mlp.experts.60.gate_proj", "model.layers.7.mlp.experts.61.gate_proj", "model.layers.7.mlp.experts.62.gate_proj", "model.layers.7.mlp.experts.63.gate_proj", "model.layers.7.mlp.experts.64.gate_proj", "model.layers.7.mlp.experts.65.gate_proj", "model.layers.7.mlp.experts.66.gate_proj", "model.layers.7.mlp.experts.67.gate_proj", "model.layers.7.mlp.experts.68.gate_proj", "model.layers.7.mlp.experts.69.gate_proj", "model.layers.7.mlp.experts.70.gate_proj", "model.layers.7.mlp.experts.71.gate_proj", "model.layers.7.mlp.experts.72.gate_proj", "model.layers.7.mlp.experts.73.gate_proj", "model.layers.7.mlp.experts.74.gate_proj", "model.layers.7.mlp.experts.75.gate_proj", "model.layers.7.mlp.experts.76.gate_proj", "model.layers.7.mlp.experts.77.gate_proj", "model.layers.7.mlp.experts.78.gate_proj", "model.layers.7.mlp.experts.79.gate_proj", "model.layers.7.mlp.experts.80.gate_proj", "model.layers.7.mlp.experts.81.gate_proj", "model.layers.7.mlp.experts.82.gate_proj", "model.layers.7.mlp.experts.83.gate_proj", "model.layers.7.mlp.experts.84.gate_proj", "model.layers.7.mlp.experts.85.gate_proj", "model.layers.7.mlp.experts.86.gate_proj", "model.layers.7.mlp.experts.87.gate_proj", "model.layers.7.mlp.experts.88.gate_proj", "model.layers.7.mlp.experts.89.gate_proj", "model.layers.7.mlp.experts.90.gate_proj", "model.layers.7.mlp.experts.91.gate_proj", "model.layers.7.mlp.experts.92.gate_proj", "model.layers.7.mlp.experts.93.gate_proj", "model.layers.7.mlp.experts.94.gate_proj", "model.layers.7.mlp.experts.95.gate_proj", "model.layers.7.mlp.experts.96.gate_proj", "model.layers.7.mlp.experts.97.gate_proj", "model.layers.7.mlp.experts.98.gate_proj", "model.layers.7.mlp.experts.99.gate_proj", "model.layers.7.mlp.experts.100.gate_proj", "model.layers.7.mlp.experts.101.gate_proj", "model.layers.7.mlp.experts.102.gate_proj", "model.layers.7.mlp.experts.103.gate_proj", "model.layers.7.mlp.experts.104.gate_proj", "model.layers.7.mlp.experts.105.gate_proj", "model.layers.7.mlp.experts.106.gate_proj", "model.layers.7.mlp.experts.107.gate_proj", "model.layers.7.mlp.experts.108.gate_proj", "model.layers.7.mlp.experts.109.gate_proj", "model.layers.7.mlp.experts.110.gate_proj", "model.layers.7.mlp.experts.111.gate_proj", "model.layers.7.mlp.experts.112.gate_proj", "model.layers.7.mlp.experts.113.gate_proj", "model.layers.7.mlp.experts.114.gate_proj", "model.layers.7.mlp.experts.115.gate_proj", "model.layers.7.mlp.experts.116.gate_proj", "model.layers.7.mlp.experts.117.gate_proj", "model.layers.7.mlp.experts.118.gate_proj", "model.layers.7.mlp.experts.119.gate_proj", "model.layers.7.mlp.experts.120.gate_proj", "model.layers.7.mlp.experts.121.gate_proj", "model.layers.7.mlp.experts.122.gate_proj", "model.layers.7.mlp.experts.123.gate_proj", "model.layers.7.mlp.experts.124.gate_proj", "model.layers.7.mlp.experts.125.gate_proj", "model.layers.7.mlp.experts.126.gate_proj", "model.layers.7.mlp.experts.127.gate_proj", "model.layers.7.mlp.experts.128.gate_proj", "model.layers.7.mlp.experts.129.gate_proj", "model.layers.7.mlp.experts.130.gate_proj", "model.layers.7.mlp.experts.131.gate_proj", "model.layers.7.mlp.experts.132.gate_proj", "model.layers.7.mlp.experts.133.gate_proj", "model.layers.7.mlp.experts.134.gate_proj", "model.layers.7.mlp.experts.135.gate_proj", "model.layers.7.mlp.experts.136.gate_proj", "model.layers.7.mlp.experts.137.gate_proj", "model.layers.7.mlp.experts.138.gate_proj", "model.layers.7.mlp.experts.139.gate_proj", "model.layers.7.mlp.experts.140.gate_proj", "model.layers.7.mlp.experts.141.gate_proj", "model.layers.7.mlp.experts.142.gate_proj", "model.layers.7.mlp.experts.143.gate_proj", "model.layers.7.mlp.experts.144.gate_proj", "model.layers.7.mlp.experts.145.gate_proj", "model.layers.7.mlp.experts.146.gate_proj", "model.layers.7.mlp.experts.147.gate_proj", "model.layers.7.mlp.experts.148.gate_proj", "model.layers.7.mlp.experts.149.gate_proj", "model.layers.7.mlp.experts.150.gate_proj", "model.layers.7.mlp.experts.151.gate_proj", "model.layers.7.mlp.experts.152.gate_proj", "model.layers.7.mlp.experts.153.gate_proj", "model.layers.7.mlp.experts.154.gate_proj", "model.layers.7.mlp.experts.155.gate_proj", "model.layers.7.mlp.experts.156.gate_proj", "model.layers.7.mlp.experts.157.gate_proj", "model.layers.7.mlp.experts.158.gate_proj", "model.layers.7.mlp.experts.159.gate_proj", "model.layers.7.mlp.experts.0.up_proj", "model.layers.7.mlp.experts.1.up_proj", "model.layers.7.mlp.experts.2.up_proj", "model.layers.7.mlp.experts.3.up_proj", "model.layers.7.mlp.experts.4.up_proj", "model.layers.7.mlp.experts.5.up_proj", "model.layers.7.mlp.experts.6.up_proj", "model.layers.7.mlp.experts.7.up_proj", "model.layers.7.mlp.experts.8.up_proj", "model.layers.7.mlp.experts.9.up_proj", "model.layers.7.mlp.experts.10.up_proj", "model.layers.7.mlp.experts.11.up_proj", "model.layers.7.mlp.experts.12.up_proj", "model.layers.7.mlp.experts.13.up_proj", "model.layers.7.mlp.experts.14.up_proj", "model.layers.7.mlp.experts.15.up_proj", "model.layers.7.mlp.experts.16.up_proj", "model.layers.7.mlp.experts.17.up_proj", "model.layers.7.mlp.experts.18.up_proj", "model.layers.7.mlp.experts.19.up_proj", "model.layers.7.mlp.experts.20.up_proj", "model.layers.7.mlp.experts.21.up_proj", "model.layers.7.mlp.experts.22.up_proj", "model.layers.7.mlp.experts.23.up_proj", "model.layers.7.mlp.experts.24.up_proj", "model.layers.7.mlp.experts.25.up_proj", "model.layers.7.mlp.experts.26.up_proj", "model.layers.7.mlp.experts.27.up_proj", "model.layers.7.mlp.experts.28.up_proj", "model.layers.7.mlp.experts.29.up_proj", "model.layers.7.mlp.experts.30.up_proj", "model.layers.7.mlp.experts.31.up_proj", "model.layers.7.mlp.experts.32.up_proj", "model.layers.7.mlp.experts.33.up_proj", "model.layers.7.mlp.experts.34.up_proj", "model.layers.7.mlp.experts.35.up_proj", "model.layers.7.mlp.experts.36.up_proj", "model.layers.7.mlp.experts.37.up_proj", "model.layers.7.mlp.experts.38.up_proj", "model.layers.7.mlp.experts.39.up_proj", "model.layers.7.mlp.experts.40.up_proj", "model.layers.7.mlp.experts.41.up_proj", "model.layers.7.mlp.experts.42.up_proj", "model.layers.7.mlp.experts.43.up_proj", "model.layers.7.mlp.experts.44.up_proj", "model.layers.7.mlp.experts.45.up_proj", "model.layers.7.mlp.experts.46.up_proj", "model.layers.7.mlp.experts.47.up_proj", "model.layers.7.mlp.experts.48.up_proj", "model.layers.7.mlp.experts.49.up_proj", "model.layers.7.mlp.experts.50.up_proj", "model.layers.7.mlp.experts.51.up_proj", "model.layers.7.mlp.experts.52.up_proj", "model.layers.7.mlp.experts.53.up_proj", "model.layers.7.mlp.experts.54.up_proj", "model.layers.7.mlp.experts.55.up_proj", "model.layers.7.mlp.experts.56.up_proj", "model.layers.7.mlp.experts.57.up_proj", "model.layers.7.mlp.experts.58.up_proj", "model.layers.7.mlp.experts.59.up_proj", "model.layers.7.mlp.experts.60.up_proj", "model.layers.7.mlp.experts.61.up_proj", "model.layers.7.mlp.experts.62.up_proj", "model.layers.7.mlp.experts.63.up_proj", "model.layers.7.mlp.experts.64.up_proj", "model.layers.7.mlp.experts.65.up_proj", "model.layers.7.mlp.experts.66.up_proj", "model.layers.7.mlp.experts.67.up_proj", "model.layers.7.mlp.experts.68.up_proj", "model.layers.7.mlp.experts.69.up_proj", "model.layers.7.mlp.experts.70.up_proj", "model.layers.7.mlp.experts.71.up_proj", "model.layers.7.mlp.experts.72.up_proj", "model.layers.7.mlp.experts.73.up_proj", "model.layers.7.mlp.experts.74.up_proj", "model.layers.7.mlp.experts.75.up_proj", "model.layers.7.mlp.experts.76.up_proj", "model.layers.7.mlp.experts.77.up_proj", "model.layers.7.mlp.experts.78.up_proj", "model.layers.7.mlp.experts.79.up_proj", "model.layers.7.mlp.experts.80.up_proj", "model.layers.7.mlp.experts.81.up_proj", "model.layers.7.mlp.experts.82.up_proj", "model.layers.7.mlp.experts.83.up_proj", "model.layers.7.mlp.experts.84.up_proj", "model.layers.7.mlp.experts.85.up_proj", "model.layers.7.mlp.experts.86.up_proj", "model.layers.7.mlp.experts.87.up_proj", "model.layers.7.mlp.experts.88.up_proj", "model.layers.7.mlp.experts.89.up_proj", "model.layers.7.mlp.experts.90.up_proj", "model.layers.7.mlp.experts.91.up_proj", "model.layers.7.mlp.experts.92.up_proj", "model.layers.7.mlp.experts.93.up_proj", "model.layers.7.mlp.experts.94.up_proj", "model.layers.7.mlp.experts.95.up_proj", "model.layers.7.mlp.experts.96.up_proj", "model.layers.7.mlp.experts.97.up_proj", "model.layers.7.mlp.experts.98.up_proj", "model.layers.7.mlp.experts.99.up_proj", "model.layers.7.mlp.experts.100.up_proj", "model.layers.7.mlp.experts.101.up_proj", "model.layers.7.mlp.experts.102.up_proj", "model.layers.7.mlp.experts.103.up_proj", "model.layers.7.mlp.experts.104.up_proj", "model.layers.7.mlp.experts.105.up_proj", "model.layers.7.mlp.experts.106.up_proj", "model.layers.7.mlp.experts.107.up_proj", "model.layers.7.mlp.experts.108.up_proj", "model.layers.7.mlp.experts.109.up_proj", "model.layers.7.mlp.experts.110.up_proj", "model.layers.7.mlp.experts.111.up_proj", "model.layers.7.mlp.experts.112.up_proj", "model.layers.7.mlp.experts.113.up_proj", "model.layers.7.mlp.experts.114.up_proj", "model.layers.7.mlp.experts.115.up_proj", "model.layers.7.mlp.experts.116.up_proj", "model.layers.7.mlp.experts.117.up_proj", "model.layers.7.mlp.experts.118.up_proj", "model.layers.7.mlp.experts.119.up_proj", "model.layers.7.mlp.experts.120.up_proj", "model.layers.7.mlp.experts.121.up_proj", "model.layers.7.mlp.experts.122.up_proj", "model.layers.7.mlp.experts.123.up_proj", "model.layers.7.mlp.experts.124.up_proj", "model.layers.7.mlp.experts.125.up_proj", "model.layers.7.mlp.experts.126.up_proj", "model.layers.7.mlp.experts.127.up_proj", "model.layers.7.mlp.experts.128.up_proj", "model.layers.7.mlp.experts.129.up_proj", "model.layers.7.mlp.experts.130.up_proj", "model.layers.7.mlp.experts.131.up_proj", "model.layers.7.mlp.experts.132.up_proj", "model.layers.7.mlp.experts.133.up_proj", "model.layers.7.mlp.experts.134.up_proj", "model.layers.7.mlp.experts.135.up_proj", "model.layers.7.mlp.experts.136.up_proj", "model.layers.7.mlp.experts.137.up_proj", "model.layers.7.mlp.experts.138.up_proj", "model.layers.7.mlp.experts.139.up_proj", "model.layers.7.mlp.experts.140.up_proj", "model.layers.7.mlp.experts.141.up_proj", "model.layers.7.mlp.experts.142.up_proj", "model.layers.7.mlp.experts.143.up_proj", "model.layers.7.mlp.experts.144.up_proj", "model.layers.7.mlp.experts.145.up_proj", "model.layers.7.mlp.experts.146.up_proj", "model.layers.7.mlp.experts.147.up_proj", "model.layers.7.mlp.experts.148.up_proj", "model.layers.7.mlp.experts.149.up_proj", "model.layers.7.mlp.experts.150.up_proj", "model.layers.7.mlp.experts.151.up_proj", "model.layers.7.mlp.experts.152.up_proj", "model.layers.7.mlp.experts.153.up_proj", "model.layers.7.mlp.experts.154.up_proj", "model.layers.7.mlp.experts.155.up_proj", "model.layers.7.mlp.experts.156.up_proj", "model.layers.7.mlp.experts.157.up_proj", "model.layers.7.mlp.experts.158.up_proj", "model.layers.7.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 6.74761249683806e-06, "dbits": 5033164800 } ] }, { "idx": 44, "layers": [ "model.layers.7.mlp.experts.0.down_proj", "model.layers.7.mlp.experts.1.down_proj", "model.layers.7.mlp.experts.2.down_proj", "model.layers.7.mlp.experts.3.down_proj", "model.layers.7.mlp.experts.4.down_proj", "model.layers.7.mlp.experts.5.down_proj", "model.layers.7.mlp.experts.6.down_proj", "model.layers.7.mlp.experts.7.down_proj", "model.layers.7.mlp.experts.8.down_proj", "model.layers.7.mlp.experts.9.down_proj", "model.layers.7.mlp.experts.10.down_proj", "model.layers.7.mlp.experts.11.down_proj", "model.layers.7.mlp.experts.12.down_proj", "model.layers.7.mlp.experts.13.down_proj", "model.layers.7.mlp.experts.14.down_proj", "model.layers.7.mlp.experts.15.down_proj", "model.layers.7.mlp.experts.16.down_proj", "model.layers.7.mlp.experts.17.down_proj", "model.layers.7.mlp.experts.18.down_proj", "model.layers.7.mlp.experts.19.down_proj", "model.layers.7.mlp.experts.20.down_proj", "model.layers.7.mlp.experts.21.down_proj", "model.layers.7.mlp.experts.22.down_proj", "model.layers.7.mlp.experts.23.down_proj", "model.layers.7.mlp.experts.24.down_proj", "model.layers.7.mlp.experts.25.down_proj", "model.layers.7.mlp.experts.26.down_proj", "model.layers.7.mlp.experts.27.down_proj", "model.layers.7.mlp.experts.28.down_proj", "model.layers.7.mlp.experts.29.down_proj", "model.layers.7.mlp.experts.30.down_proj", "model.layers.7.mlp.experts.31.down_proj", "model.layers.7.mlp.experts.32.down_proj", "model.layers.7.mlp.experts.33.down_proj", "model.layers.7.mlp.experts.34.down_proj", "model.layers.7.mlp.experts.35.down_proj", "model.layers.7.mlp.experts.36.down_proj", "model.layers.7.mlp.experts.37.down_proj", "model.layers.7.mlp.experts.38.down_proj", "model.layers.7.mlp.experts.39.down_proj", "model.layers.7.mlp.experts.40.down_proj", "model.layers.7.mlp.experts.41.down_proj", "model.layers.7.mlp.experts.42.down_proj", "model.layers.7.mlp.experts.43.down_proj", "model.layers.7.mlp.experts.44.down_proj", "model.layers.7.mlp.experts.45.down_proj", "model.layers.7.mlp.experts.46.down_proj", "model.layers.7.mlp.experts.47.down_proj", "model.layers.7.mlp.experts.48.down_proj", "model.layers.7.mlp.experts.49.down_proj", "model.layers.7.mlp.experts.50.down_proj", "model.layers.7.mlp.experts.51.down_proj", "model.layers.7.mlp.experts.52.down_proj", "model.layers.7.mlp.experts.53.down_proj", "model.layers.7.mlp.experts.54.down_proj", "model.layers.7.mlp.experts.55.down_proj", "model.layers.7.mlp.experts.56.down_proj", "model.layers.7.mlp.experts.57.down_proj", "model.layers.7.mlp.experts.58.down_proj", "model.layers.7.mlp.experts.59.down_proj", "model.layers.7.mlp.experts.60.down_proj", "model.layers.7.mlp.experts.61.down_proj", "model.layers.7.mlp.experts.62.down_proj", "model.layers.7.mlp.experts.63.down_proj", "model.layers.7.mlp.experts.64.down_proj", "model.layers.7.mlp.experts.65.down_proj", "model.layers.7.mlp.experts.66.down_proj", "model.layers.7.mlp.experts.67.down_proj", "model.layers.7.mlp.experts.68.down_proj", "model.layers.7.mlp.experts.69.down_proj", "model.layers.7.mlp.experts.70.down_proj", "model.layers.7.mlp.experts.71.down_proj", "model.layers.7.mlp.experts.72.down_proj", "model.layers.7.mlp.experts.73.down_proj", "model.layers.7.mlp.experts.74.down_proj", "model.layers.7.mlp.experts.75.down_proj", "model.layers.7.mlp.experts.76.down_proj", "model.layers.7.mlp.experts.77.down_proj", "model.layers.7.mlp.experts.78.down_proj", "model.layers.7.mlp.experts.79.down_proj", "model.layers.7.mlp.experts.80.down_proj", "model.layers.7.mlp.experts.81.down_proj", "model.layers.7.mlp.experts.82.down_proj", "model.layers.7.mlp.experts.83.down_proj", "model.layers.7.mlp.experts.84.down_proj", "model.layers.7.mlp.experts.85.down_proj", "model.layers.7.mlp.experts.86.down_proj", "model.layers.7.mlp.experts.87.down_proj", "model.layers.7.mlp.experts.88.down_proj", "model.layers.7.mlp.experts.89.down_proj", "model.layers.7.mlp.experts.90.down_proj", "model.layers.7.mlp.experts.91.down_proj", "model.layers.7.mlp.experts.92.down_proj", "model.layers.7.mlp.experts.93.down_proj", "model.layers.7.mlp.experts.94.down_proj", "model.layers.7.mlp.experts.95.down_proj", "model.layers.7.mlp.experts.96.down_proj", "model.layers.7.mlp.experts.97.down_proj", "model.layers.7.mlp.experts.98.down_proj", "model.layers.7.mlp.experts.99.down_proj", "model.layers.7.mlp.experts.100.down_proj", "model.layers.7.mlp.experts.101.down_proj", "model.layers.7.mlp.experts.102.down_proj", "model.layers.7.mlp.experts.103.down_proj", "model.layers.7.mlp.experts.104.down_proj", "model.layers.7.mlp.experts.105.down_proj", "model.layers.7.mlp.experts.106.down_proj", "model.layers.7.mlp.experts.107.down_proj", "model.layers.7.mlp.experts.108.down_proj", "model.layers.7.mlp.experts.109.down_proj", "model.layers.7.mlp.experts.110.down_proj", "model.layers.7.mlp.experts.111.down_proj", "model.layers.7.mlp.experts.112.down_proj", "model.layers.7.mlp.experts.113.down_proj", "model.layers.7.mlp.experts.114.down_proj", "model.layers.7.mlp.experts.115.down_proj", "model.layers.7.mlp.experts.116.down_proj", "model.layers.7.mlp.experts.117.down_proj", "model.layers.7.mlp.experts.118.down_proj", "model.layers.7.mlp.experts.119.down_proj", "model.layers.7.mlp.experts.120.down_proj", "model.layers.7.mlp.experts.121.down_proj", "model.layers.7.mlp.experts.122.down_proj", "model.layers.7.mlp.experts.123.down_proj", "model.layers.7.mlp.experts.124.down_proj", "model.layers.7.mlp.experts.125.down_proj", "model.layers.7.mlp.experts.126.down_proj", "model.layers.7.mlp.experts.127.down_proj", "model.layers.7.mlp.experts.128.down_proj", "model.layers.7.mlp.experts.129.down_proj", "model.layers.7.mlp.experts.130.down_proj", "model.layers.7.mlp.experts.131.down_proj", "model.layers.7.mlp.experts.132.down_proj", "model.layers.7.mlp.experts.133.down_proj", "model.layers.7.mlp.experts.134.down_proj", "model.layers.7.mlp.experts.135.down_proj", "model.layers.7.mlp.experts.136.down_proj", "model.layers.7.mlp.experts.137.down_proj", "model.layers.7.mlp.experts.138.down_proj", "model.layers.7.mlp.experts.139.down_proj", "model.layers.7.mlp.experts.140.down_proj", "model.layers.7.mlp.experts.141.down_proj", "model.layers.7.mlp.experts.142.down_proj", "model.layers.7.mlp.experts.143.down_proj", "model.layers.7.mlp.experts.144.down_proj", "model.layers.7.mlp.experts.145.down_proj", "model.layers.7.mlp.experts.146.down_proj", "model.layers.7.mlp.experts.147.down_proj", "model.layers.7.mlp.experts.148.down_proj", "model.layers.7.mlp.experts.149.down_proj", "model.layers.7.mlp.experts.150.down_proj", "model.layers.7.mlp.experts.151.down_proj", "model.layers.7.mlp.experts.152.down_proj", "model.layers.7.mlp.experts.153.down_proj", "model.layers.7.mlp.experts.154.down_proj", "model.layers.7.mlp.experts.155.down_proj", "model.layers.7.mlp.experts.156.down_proj", "model.layers.7.mlp.experts.157.down_proj", "model.layers.7.mlp.experts.158.down_proj", "model.layers.7.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -3.2835843740031557e-06, "dbits": 2516582400 } ] }, { "idx": 45, "layers": [ "model.layers.8.self_attn.q_proj" ], "candidates": [ { "dkld": -4.600849933921857e-07, "dbits": 125829120 } ] }, { "idx": 46, "layers": [ "model.layers.8.self_attn.k_proj", "model.layers.8.self_attn.v_proj" ], "candidates": [ { "dkld": -2.218371373601262e-05, "dbits": 20971520 } ] }, { "idx": 47, "layers": [ "model.layers.8.self_attn.o_proj" ], "candidates": [ { "dkld": -1.330414670519509e-06, "dbits": 125829120 } ] }, { "idx": 48, "layers": [ "model.layers.8.mlp.shared_experts.gate_proj", "model.layers.8.mlp.shared_experts.up_proj", "model.layers.8.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.4476230717264116e-05, "dbits": 47185920 } ] }, { "idx": 49, "layers": [ "model.layers.8.mlp.experts.0.gate_proj", "model.layers.8.mlp.experts.1.gate_proj", "model.layers.8.mlp.experts.2.gate_proj", "model.layers.8.mlp.experts.3.gate_proj", "model.layers.8.mlp.experts.4.gate_proj", "model.layers.8.mlp.experts.5.gate_proj", "model.layers.8.mlp.experts.6.gate_proj", "model.layers.8.mlp.experts.7.gate_proj", "model.layers.8.mlp.experts.8.gate_proj", "model.layers.8.mlp.experts.9.gate_proj", "model.layers.8.mlp.experts.10.gate_proj", "model.layers.8.mlp.experts.11.gate_proj", "model.layers.8.mlp.experts.12.gate_proj", "model.layers.8.mlp.experts.13.gate_proj", "model.layers.8.mlp.experts.14.gate_proj", "model.layers.8.mlp.experts.15.gate_proj", "model.layers.8.mlp.experts.16.gate_proj", "model.layers.8.mlp.experts.17.gate_proj", "model.layers.8.mlp.experts.18.gate_proj", "model.layers.8.mlp.experts.19.gate_proj", "model.layers.8.mlp.experts.20.gate_proj", "model.layers.8.mlp.experts.21.gate_proj", "model.layers.8.mlp.experts.22.gate_proj", "model.layers.8.mlp.experts.23.gate_proj", "model.layers.8.mlp.experts.24.gate_proj", "model.layers.8.mlp.experts.25.gate_proj", "model.layers.8.mlp.experts.26.gate_proj", "model.layers.8.mlp.experts.27.gate_proj", "model.layers.8.mlp.experts.28.gate_proj", "model.layers.8.mlp.experts.29.gate_proj", "model.layers.8.mlp.experts.30.gate_proj", "model.layers.8.mlp.experts.31.gate_proj", "model.layers.8.mlp.experts.32.gate_proj", "model.layers.8.mlp.experts.33.gate_proj", "model.layers.8.mlp.experts.34.gate_proj", "model.layers.8.mlp.experts.35.gate_proj", "model.layers.8.mlp.experts.36.gate_proj", "model.layers.8.mlp.experts.37.gate_proj", "model.layers.8.mlp.experts.38.gate_proj", "model.layers.8.mlp.experts.39.gate_proj", "model.layers.8.mlp.experts.40.gate_proj", "model.layers.8.mlp.experts.41.gate_proj", "model.layers.8.mlp.experts.42.gate_proj", "model.layers.8.mlp.experts.43.gate_proj", "model.layers.8.mlp.experts.44.gate_proj", "model.layers.8.mlp.experts.45.gate_proj", "model.layers.8.mlp.experts.46.gate_proj", "model.layers.8.mlp.experts.47.gate_proj", "model.layers.8.mlp.experts.48.gate_proj", "model.layers.8.mlp.experts.49.gate_proj", "model.layers.8.mlp.experts.50.gate_proj", "model.layers.8.mlp.experts.51.gate_proj", "model.layers.8.mlp.experts.52.gate_proj", "model.layers.8.mlp.experts.53.gate_proj", "model.layers.8.mlp.experts.54.gate_proj", "model.layers.8.mlp.experts.55.gate_proj", "model.layers.8.mlp.experts.56.gate_proj", "model.layers.8.mlp.experts.57.gate_proj", "model.layers.8.mlp.experts.58.gate_proj", "model.layers.8.mlp.experts.59.gate_proj", "model.layers.8.mlp.experts.60.gate_proj", "model.layers.8.mlp.experts.61.gate_proj", "model.layers.8.mlp.experts.62.gate_proj", "model.layers.8.mlp.experts.63.gate_proj", "model.layers.8.mlp.experts.64.gate_proj", "model.layers.8.mlp.experts.65.gate_proj", "model.layers.8.mlp.experts.66.gate_proj", "model.layers.8.mlp.experts.67.gate_proj", "model.layers.8.mlp.experts.68.gate_proj", "model.layers.8.mlp.experts.69.gate_proj", "model.layers.8.mlp.experts.70.gate_proj", "model.layers.8.mlp.experts.71.gate_proj", "model.layers.8.mlp.experts.72.gate_proj", "model.layers.8.mlp.experts.73.gate_proj", "model.layers.8.mlp.experts.74.gate_proj", "model.layers.8.mlp.experts.75.gate_proj", "model.layers.8.mlp.experts.76.gate_proj", "model.layers.8.mlp.experts.77.gate_proj", "model.layers.8.mlp.experts.78.gate_proj", "model.layers.8.mlp.experts.79.gate_proj", "model.layers.8.mlp.experts.80.gate_proj", "model.layers.8.mlp.experts.81.gate_proj", "model.layers.8.mlp.experts.82.gate_proj", "model.layers.8.mlp.experts.83.gate_proj", "model.layers.8.mlp.experts.84.gate_proj", "model.layers.8.mlp.experts.85.gate_proj", "model.layers.8.mlp.experts.86.gate_proj", "model.layers.8.mlp.experts.87.gate_proj", "model.layers.8.mlp.experts.88.gate_proj", "model.layers.8.mlp.experts.89.gate_proj", "model.layers.8.mlp.experts.90.gate_proj", "model.layers.8.mlp.experts.91.gate_proj", "model.layers.8.mlp.experts.92.gate_proj", "model.layers.8.mlp.experts.93.gate_proj", "model.layers.8.mlp.experts.94.gate_proj", "model.layers.8.mlp.experts.95.gate_proj", "model.layers.8.mlp.experts.96.gate_proj", "model.layers.8.mlp.experts.97.gate_proj", "model.layers.8.mlp.experts.98.gate_proj", "model.layers.8.mlp.experts.99.gate_proj", "model.layers.8.mlp.experts.100.gate_proj", "model.layers.8.mlp.experts.101.gate_proj", "model.layers.8.mlp.experts.102.gate_proj", "model.layers.8.mlp.experts.103.gate_proj", "model.layers.8.mlp.experts.104.gate_proj", "model.layers.8.mlp.experts.105.gate_proj", "model.layers.8.mlp.experts.106.gate_proj", "model.layers.8.mlp.experts.107.gate_proj", "model.layers.8.mlp.experts.108.gate_proj", "model.layers.8.mlp.experts.109.gate_proj", "model.layers.8.mlp.experts.110.gate_proj", "model.layers.8.mlp.experts.111.gate_proj", "model.layers.8.mlp.experts.112.gate_proj", "model.layers.8.mlp.experts.113.gate_proj", "model.layers.8.mlp.experts.114.gate_proj", "model.layers.8.mlp.experts.115.gate_proj", "model.layers.8.mlp.experts.116.gate_proj", "model.layers.8.mlp.experts.117.gate_proj", "model.layers.8.mlp.experts.118.gate_proj", "model.layers.8.mlp.experts.119.gate_proj", "model.layers.8.mlp.experts.120.gate_proj", "model.layers.8.mlp.experts.121.gate_proj", "model.layers.8.mlp.experts.122.gate_proj", "model.layers.8.mlp.experts.123.gate_proj", "model.layers.8.mlp.experts.124.gate_proj", "model.layers.8.mlp.experts.125.gate_proj", "model.layers.8.mlp.experts.126.gate_proj", "model.layers.8.mlp.experts.127.gate_proj", "model.layers.8.mlp.experts.128.gate_proj", "model.layers.8.mlp.experts.129.gate_proj", "model.layers.8.mlp.experts.130.gate_proj", "model.layers.8.mlp.experts.131.gate_proj", "model.layers.8.mlp.experts.132.gate_proj", "model.layers.8.mlp.experts.133.gate_proj", "model.layers.8.mlp.experts.134.gate_proj", "model.layers.8.mlp.experts.135.gate_proj", "model.layers.8.mlp.experts.136.gate_proj", "model.layers.8.mlp.experts.137.gate_proj", "model.layers.8.mlp.experts.138.gate_proj", "model.layers.8.mlp.experts.139.gate_proj", "model.layers.8.mlp.experts.140.gate_proj", "model.layers.8.mlp.experts.141.gate_proj", "model.layers.8.mlp.experts.142.gate_proj", "model.layers.8.mlp.experts.143.gate_proj", "model.layers.8.mlp.experts.144.gate_proj", "model.layers.8.mlp.experts.145.gate_proj", "model.layers.8.mlp.experts.146.gate_proj", "model.layers.8.mlp.experts.147.gate_proj", "model.layers.8.mlp.experts.148.gate_proj", "model.layers.8.mlp.experts.149.gate_proj", "model.layers.8.mlp.experts.150.gate_proj", "model.layers.8.mlp.experts.151.gate_proj", "model.layers.8.mlp.experts.152.gate_proj", "model.layers.8.mlp.experts.153.gate_proj", "model.layers.8.mlp.experts.154.gate_proj", "model.layers.8.mlp.experts.155.gate_proj", "model.layers.8.mlp.experts.156.gate_proj", "model.layers.8.mlp.experts.157.gate_proj", "model.layers.8.mlp.experts.158.gate_proj", "model.layers.8.mlp.experts.159.gate_proj", "model.layers.8.mlp.experts.0.up_proj", "model.layers.8.mlp.experts.1.up_proj", "model.layers.8.mlp.experts.2.up_proj", "model.layers.8.mlp.experts.3.up_proj", "model.layers.8.mlp.experts.4.up_proj", "model.layers.8.mlp.experts.5.up_proj", "model.layers.8.mlp.experts.6.up_proj", "model.layers.8.mlp.experts.7.up_proj", "model.layers.8.mlp.experts.8.up_proj", "model.layers.8.mlp.experts.9.up_proj", "model.layers.8.mlp.experts.10.up_proj", "model.layers.8.mlp.experts.11.up_proj", "model.layers.8.mlp.experts.12.up_proj", "model.layers.8.mlp.experts.13.up_proj", "model.layers.8.mlp.experts.14.up_proj", "model.layers.8.mlp.experts.15.up_proj", "model.layers.8.mlp.experts.16.up_proj", "model.layers.8.mlp.experts.17.up_proj", "model.layers.8.mlp.experts.18.up_proj", "model.layers.8.mlp.experts.19.up_proj", "model.layers.8.mlp.experts.20.up_proj", "model.layers.8.mlp.experts.21.up_proj", "model.layers.8.mlp.experts.22.up_proj", "model.layers.8.mlp.experts.23.up_proj", "model.layers.8.mlp.experts.24.up_proj", "model.layers.8.mlp.experts.25.up_proj", "model.layers.8.mlp.experts.26.up_proj", "model.layers.8.mlp.experts.27.up_proj", "model.layers.8.mlp.experts.28.up_proj", "model.layers.8.mlp.experts.29.up_proj", "model.layers.8.mlp.experts.30.up_proj", "model.layers.8.mlp.experts.31.up_proj", "model.layers.8.mlp.experts.32.up_proj", "model.layers.8.mlp.experts.33.up_proj", "model.layers.8.mlp.experts.34.up_proj", "model.layers.8.mlp.experts.35.up_proj", "model.layers.8.mlp.experts.36.up_proj", "model.layers.8.mlp.experts.37.up_proj", "model.layers.8.mlp.experts.38.up_proj", "model.layers.8.mlp.experts.39.up_proj", "model.layers.8.mlp.experts.40.up_proj", "model.layers.8.mlp.experts.41.up_proj", "model.layers.8.mlp.experts.42.up_proj", "model.layers.8.mlp.experts.43.up_proj", "model.layers.8.mlp.experts.44.up_proj", "model.layers.8.mlp.experts.45.up_proj", "model.layers.8.mlp.experts.46.up_proj", "model.layers.8.mlp.experts.47.up_proj", "model.layers.8.mlp.experts.48.up_proj", "model.layers.8.mlp.experts.49.up_proj", "model.layers.8.mlp.experts.50.up_proj", "model.layers.8.mlp.experts.51.up_proj", "model.layers.8.mlp.experts.52.up_proj", "model.layers.8.mlp.experts.53.up_proj", "model.layers.8.mlp.experts.54.up_proj", "model.layers.8.mlp.experts.55.up_proj", "model.layers.8.mlp.experts.56.up_proj", "model.layers.8.mlp.experts.57.up_proj", "model.layers.8.mlp.experts.58.up_proj", "model.layers.8.mlp.experts.59.up_proj", "model.layers.8.mlp.experts.60.up_proj", "model.layers.8.mlp.experts.61.up_proj", "model.layers.8.mlp.experts.62.up_proj", "model.layers.8.mlp.experts.63.up_proj", "model.layers.8.mlp.experts.64.up_proj", "model.layers.8.mlp.experts.65.up_proj", "model.layers.8.mlp.experts.66.up_proj", "model.layers.8.mlp.experts.67.up_proj", "model.layers.8.mlp.experts.68.up_proj", "model.layers.8.mlp.experts.69.up_proj", "model.layers.8.mlp.experts.70.up_proj", "model.layers.8.mlp.experts.71.up_proj", "model.layers.8.mlp.experts.72.up_proj", "model.layers.8.mlp.experts.73.up_proj", "model.layers.8.mlp.experts.74.up_proj", "model.layers.8.mlp.experts.75.up_proj", "model.layers.8.mlp.experts.76.up_proj", "model.layers.8.mlp.experts.77.up_proj", "model.layers.8.mlp.experts.78.up_proj", "model.layers.8.mlp.experts.79.up_proj", "model.layers.8.mlp.experts.80.up_proj", "model.layers.8.mlp.experts.81.up_proj", "model.layers.8.mlp.experts.82.up_proj", "model.layers.8.mlp.experts.83.up_proj", "model.layers.8.mlp.experts.84.up_proj", "model.layers.8.mlp.experts.85.up_proj", "model.layers.8.mlp.experts.86.up_proj", "model.layers.8.mlp.experts.87.up_proj", "model.layers.8.mlp.experts.88.up_proj", "model.layers.8.mlp.experts.89.up_proj", "model.layers.8.mlp.experts.90.up_proj", "model.layers.8.mlp.experts.91.up_proj", "model.layers.8.mlp.experts.92.up_proj", "model.layers.8.mlp.experts.93.up_proj", "model.layers.8.mlp.experts.94.up_proj", "model.layers.8.mlp.experts.95.up_proj", "model.layers.8.mlp.experts.96.up_proj", "model.layers.8.mlp.experts.97.up_proj", "model.layers.8.mlp.experts.98.up_proj", "model.layers.8.mlp.experts.99.up_proj", "model.layers.8.mlp.experts.100.up_proj", "model.layers.8.mlp.experts.101.up_proj", "model.layers.8.mlp.experts.102.up_proj", "model.layers.8.mlp.experts.103.up_proj", "model.layers.8.mlp.experts.104.up_proj", "model.layers.8.mlp.experts.105.up_proj", "model.layers.8.mlp.experts.106.up_proj", "model.layers.8.mlp.experts.107.up_proj", "model.layers.8.mlp.experts.108.up_proj", "model.layers.8.mlp.experts.109.up_proj", "model.layers.8.mlp.experts.110.up_proj", "model.layers.8.mlp.experts.111.up_proj", "model.layers.8.mlp.experts.112.up_proj", "model.layers.8.mlp.experts.113.up_proj", "model.layers.8.mlp.experts.114.up_proj", "model.layers.8.mlp.experts.115.up_proj", "model.layers.8.mlp.experts.116.up_proj", "model.layers.8.mlp.experts.117.up_proj", "model.layers.8.mlp.experts.118.up_proj", "model.layers.8.mlp.experts.119.up_proj", "model.layers.8.mlp.experts.120.up_proj", "model.layers.8.mlp.experts.121.up_proj", "model.layers.8.mlp.experts.122.up_proj", "model.layers.8.mlp.experts.123.up_proj", "model.layers.8.mlp.experts.124.up_proj", "model.layers.8.mlp.experts.125.up_proj", "model.layers.8.mlp.experts.126.up_proj", "model.layers.8.mlp.experts.127.up_proj", "model.layers.8.mlp.experts.128.up_proj", "model.layers.8.mlp.experts.129.up_proj", "model.layers.8.mlp.experts.130.up_proj", "model.layers.8.mlp.experts.131.up_proj", "model.layers.8.mlp.experts.132.up_proj", "model.layers.8.mlp.experts.133.up_proj", "model.layers.8.mlp.experts.134.up_proj", "model.layers.8.mlp.experts.135.up_proj", "model.layers.8.mlp.experts.136.up_proj", "model.layers.8.mlp.experts.137.up_proj", "model.layers.8.mlp.experts.138.up_proj", "model.layers.8.mlp.experts.139.up_proj", "model.layers.8.mlp.experts.140.up_proj", "model.layers.8.mlp.experts.141.up_proj", "model.layers.8.mlp.experts.142.up_proj", "model.layers.8.mlp.experts.143.up_proj", "model.layers.8.mlp.experts.144.up_proj", "model.layers.8.mlp.experts.145.up_proj", "model.layers.8.mlp.experts.146.up_proj", "model.layers.8.mlp.experts.147.up_proj", "model.layers.8.mlp.experts.148.up_proj", "model.layers.8.mlp.experts.149.up_proj", "model.layers.8.mlp.experts.150.up_proj", "model.layers.8.mlp.experts.151.up_proj", "model.layers.8.mlp.experts.152.up_proj", "model.layers.8.mlp.experts.153.up_proj", "model.layers.8.mlp.experts.154.up_proj", "model.layers.8.mlp.experts.155.up_proj", "model.layers.8.mlp.experts.156.up_proj", "model.layers.8.mlp.experts.157.up_proj", "model.layers.8.mlp.experts.158.up_proj", "model.layers.8.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 7.808403461240319e-06, "dbits": 5033164800 } ] }, { "idx": 50, "layers": [ "model.layers.8.mlp.experts.0.down_proj", "model.layers.8.mlp.experts.1.down_proj", "model.layers.8.mlp.experts.2.down_proj", "model.layers.8.mlp.experts.3.down_proj", "model.layers.8.mlp.experts.4.down_proj", "model.layers.8.mlp.experts.5.down_proj", "model.layers.8.mlp.experts.6.down_proj", "model.layers.8.mlp.experts.7.down_proj", "model.layers.8.mlp.experts.8.down_proj", "model.layers.8.mlp.experts.9.down_proj", "model.layers.8.mlp.experts.10.down_proj", "model.layers.8.mlp.experts.11.down_proj", "model.layers.8.mlp.experts.12.down_proj", "model.layers.8.mlp.experts.13.down_proj", "model.layers.8.mlp.experts.14.down_proj", "model.layers.8.mlp.experts.15.down_proj", "model.layers.8.mlp.experts.16.down_proj", "model.layers.8.mlp.experts.17.down_proj", "model.layers.8.mlp.experts.18.down_proj", "model.layers.8.mlp.experts.19.down_proj", "model.layers.8.mlp.experts.20.down_proj", "model.layers.8.mlp.experts.21.down_proj", "model.layers.8.mlp.experts.22.down_proj", "model.layers.8.mlp.experts.23.down_proj", "model.layers.8.mlp.experts.24.down_proj", "model.layers.8.mlp.experts.25.down_proj", "model.layers.8.mlp.experts.26.down_proj", "model.layers.8.mlp.experts.27.down_proj", "model.layers.8.mlp.experts.28.down_proj", "model.layers.8.mlp.experts.29.down_proj", "model.layers.8.mlp.experts.30.down_proj", "model.layers.8.mlp.experts.31.down_proj", "model.layers.8.mlp.experts.32.down_proj", "model.layers.8.mlp.experts.33.down_proj", "model.layers.8.mlp.experts.34.down_proj", "model.layers.8.mlp.experts.35.down_proj", "model.layers.8.mlp.experts.36.down_proj", "model.layers.8.mlp.experts.37.down_proj", "model.layers.8.mlp.experts.38.down_proj", "model.layers.8.mlp.experts.39.down_proj", "model.layers.8.mlp.experts.40.down_proj", "model.layers.8.mlp.experts.41.down_proj", "model.layers.8.mlp.experts.42.down_proj", "model.layers.8.mlp.experts.43.down_proj", "model.layers.8.mlp.experts.44.down_proj", "model.layers.8.mlp.experts.45.down_proj", "model.layers.8.mlp.experts.46.down_proj", "model.layers.8.mlp.experts.47.down_proj", "model.layers.8.mlp.experts.48.down_proj", "model.layers.8.mlp.experts.49.down_proj", "model.layers.8.mlp.experts.50.down_proj", "model.layers.8.mlp.experts.51.down_proj", "model.layers.8.mlp.experts.52.down_proj", "model.layers.8.mlp.experts.53.down_proj", "model.layers.8.mlp.experts.54.down_proj", "model.layers.8.mlp.experts.55.down_proj", "model.layers.8.mlp.experts.56.down_proj", "model.layers.8.mlp.experts.57.down_proj", "model.layers.8.mlp.experts.58.down_proj", "model.layers.8.mlp.experts.59.down_proj", "model.layers.8.mlp.experts.60.down_proj", "model.layers.8.mlp.experts.61.down_proj", "model.layers.8.mlp.experts.62.down_proj", "model.layers.8.mlp.experts.63.down_proj", "model.layers.8.mlp.experts.64.down_proj", "model.layers.8.mlp.experts.65.down_proj", "model.layers.8.mlp.experts.66.down_proj", "model.layers.8.mlp.experts.67.down_proj", "model.layers.8.mlp.experts.68.down_proj", "model.layers.8.mlp.experts.69.down_proj", "model.layers.8.mlp.experts.70.down_proj", "model.layers.8.mlp.experts.71.down_proj", "model.layers.8.mlp.experts.72.down_proj", "model.layers.8.mlp.experts.73.down_proj", "model.layers.8.mlp.experts.74.down_proj", "model.layers.8.mlp.experts.75.down_proj", "model.layers.8.mlp.experts.76.down_proj", "model.layers.8.mlp.experts.77.down_proj", "model.layers.8.mlp.experts.78.down_proj", "model.layers.8.mlp.experts.79.down_proj", "model.layers.8.mlp.experts.80.down_proj", "model.layers.8.mlp.experts.81.down_proj", "model.layers.8.mlp.experts.82.down_proj", "model.layers.8.mlp.experts.83.down_proj", "model.layers.8.mlp.experts.84.down_proj", "model.layers.8.mlp.experts.85.down_proj", "model.layers.8.mlp.experts.86.down_proj", "model.layers.8.mlp.experts.87.down_proj", "model.layers.8.mlp.experts.88.down_proj", "model.layers.8.mlp.experts.89.down_proj", "model.layers.8.mlp.experts.90.down_proj", "model.layers.8.mlp.experts.91.down_proj", "model.layers.8.mlp.experts.92.down_proj", "model.layers.8.mlp.experts.93.down_proj", "model.layers.8.mlp.experts.94.down_proj", "model.layers.8.mlp.experts.95.down_proj", "model.layers.8.mlp.experts.96.down_proj", "model.layers.8.mlp.experts.97.down_proj", "model.layers.8.mlp.experts.98.down_proj", "model.layers.8.mlp.experts.99.down_proj", "model.layers.8.mlp.experts.100.down_proj", "model.layers.8.mlp.experts.101.down_proj", "model.layers.8.mlp.experts.102.down_proj", "model.layers.8.mlp.experts.103.down_proj", "model.layers.8.mlp.experts.104.down_proj", "model.layers.8.mlp.experts.105.down_proj", "model.layers.8.mlp.experts.106.down_proj", "model.layers.8.mlp.experts.107.down_proj", "model.layers.8.mlp.experts.108.down_proj", "model.layers.8.mlp.experts.109.down_proj", "model.layers.8.mlp.experts.110.down_proj", "model.layers.8.mlp.experts.111.down_proj", "model.layers.8.mlp.experts.112.down_proj", "model.layers.8.mlp.experts.113.down_proj", "model.layers.8.mlp.experts.114.down_proj", "model.layers.8.mlp.experts.115.down_proj", "model.layers.8.mlp.experts.116.down_proj", "model.layers.8.mlp.experts.117.down_proj", "model.layers.8.mlp.experts.118.down_proj", "model.layers.8.mlp.experts.119.down_proj", "model.layers.8.mlp.experts.120.down_proj", "model.layers.8.mlp.experts.121.down_proj", "model.layers.8.mlp.experts.122.down_proj", "model.layers.8.mlp.experts.123.down_proj", "model.layers.8.mlp.experts.124.down_proj", "model.layers.8.mlp.experts.125.down_proj", "model.layers.8.mlp.experts.126.down_proj", "model.layers.8.mlp.experts.127.down_proj", "model.layers.8.mlp.experts.128.down_proj", "model.layers.8.mlp.experts.129.down_proj", "model.layers.8.mlp.experts.130.down_proj", "model.layers.8.mlp.experts.131.down_proj", "model.layers.8.mlp.experts.132.down_proj", "model.layers.8.mlp.experts.133.down_proj", "model.layers.8.mlp.experts.134.down_proj", "model.layers.8.mlp.experts.135.down_proj", "model.layers.8.mlp.experts.136.down_proj", "model.layers.8.mlp.experts.137.down_proj", "model.layers.8.mlp.experts.138.down_proj", "model.layers.8.mlp.experts.139.down_proj", "model.layers.8.mlp.experts.140.down_proj", "model.layers.8.mlp.experts.141.down_proj", "model.layers.8.mlp.experts.142.down_proj", "model.layers.8.mlp.experts.143.down_proj", "model.layers.8.mlp.experts.144.down_proj", "model.layers.8.mlp.experts.145.down_proj", "model.layers.8.mlp.experts.146.down_proj", "model.layers.8.mlp.experts.147.down_proj", "model.layers.8.mlp.experts.148.down_proj", "model.layers.8.mlp.experts.149.down_proj", "model.layers.8.mlp.experts.150.down_proj", "model.layers.8.mlp.experts.151.down_proj", "model.layers.8.mlp.experts.152.down_proj", "model.layers.8.mlp.experts.153.down_proj", "model.layers.8.mlp.experts.154.down_proj", "model.layers.8.mlp.experts.155.down_proj", "model.layers.8.mlp.experts.156.down_proj", "model.layers.8.mlp.experts.157.down_proj", "model.layers.8.mlp.experts.158.down_proj", "model.layers.8.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -2.9296075808815658e-05, "dbits": 2516582400 } ] }, { "idx": 51, "layers": [ "model.layers.9.self_attn.q_proj" ], "candidates": [ { "dkld": 1.736813865136364e-05, "dbits": 125829120 } ] }, { "idx": 52, "layers": [ "model.layers.9.self_attn.k_proj", "model.layers.9.self_attn.v_proj" ], "candidates": [ { "dkld": -2.877906954381617e-05, "dbits": 20971520 } ] }, { "idx": 53, "layers": [ "model.layers.9.self_attn.o_proj" ], "candidates": [ { "dkld": -2.1711978479288475e-05, "dbits": 125829120 } ] }, { "idx": 54, "layers": [ "model.layers.9.mlp.shared_experts.gate_proj", "model.layers.9.mlp.shared_experts.up_proj", "model.layers.9.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.0827144403014944e-07, "dbits": 47185920 } ] }, { "idx": 55, "layers": [ "model.layers.9.mlp.experts.0.gate_proj", "model.layers.9.mlp.experts.1.gate_proj", "model.layers.9.mlp.experts.2.gate_proj", "model.layers.9.mlp.experts.3.gate_proj", "model.layers.9.mlp.experts.4.gate_proj", "model.layers.9.mlp.experts.5.gate_proj", "model.layers.9.mlp.experts.6.gate_proj", "model.layers.9.mlp.experts.7.gate_proj", "model.layers.9.mlp.experts.8.gate_proj", "model.layers.9.mlp.experts.9.gate_proj", "model.layers.9.mlp.experts.10.gate_proj", "model.layers.9.mlp.experts.11.gate_proj", "model.layers.9.mlp.experts.12.gate_proj", "model.layers.9.mlp.experts.13.gate_proj", "model.layers.9.mlp.experts.14.gate_proj", "model.layers.9.mlp.experts.15.gate_proj", "model.layers.9.mlp.experts.16.gate_proj", "model.layers.9.mlp.experts.17.gate_proj", "model.layers.9.mlp.experts.18.gate_proj", "model.layers.9.mlp.experts.19.gate_proj", "model.layers.9.mlp.experts.20.gate_proj", "model.layers.9.mlp.experts.21.gate_proj", "model.layers.9.mlp.experts.22.gate_proj", "model.layers.9.mlp.experts.23.gate_proj", "model.layers.9.mlp.experts.24.gate_proj", "model.layers.9.mlp.experts.25.gate_proj", "model.layers.9.mlp.experts.26.gate_proj", "model.layers.9.mlp.experts.27.gate_proj", "model.layers.9.mlp.experts.28.gate_proj", "model.layers.9.mlp.experts.29.gate_proj", "model.layers.9.mlp.experts.30.gate_proj", "model.layers.9.mlp.experts.31.gate_proj", "model.layers.9.mlp.experts.32.gate_proj", "model.layers.9.mlp.experts.33.gate_proj", "model.layers.9.mlp.experts.34.gate_proj", "model.layers.9.mlp.experts.35.gate_proj", "model.layers.9.mlp.experts.36.gate_proj", "model.layers.9.mlp.experts.37.gate_proj", "model.layers.9.mlp.experts.38.gate_proj", "model.layers.9.mlp.experts.39.gate_proj", "model.layers.9.mlp.experts.40.gate_proj", "model.layers.9.mlp.experts.41.gate_proj", "model.layers.9.mlp.experts.42.gate_proj", "model.layers.9.mlp.experts.43.gate_proj", "model.layers.9.mlp.experts.44.gate_proj", "model.layers.9.mlp.experts.45.gate_proj", "model.layers.9.mlp.experts.46.gate_proj", "model.layers.9.mlp.experts.47.gate_proj", "model.layers.9.mlp.experts.48.gate_proj", "model.layers.9.mlp.experts.49.gate_proj", "model.layers.9.mlp.experts.50.gate_proj", "model.layers.9.mlp.experts.51.gate_proj", "model.layers.9.mlp.experts.52.gate_proj", "model.layers.9.mlp.experts.53.gate_proj", "model.layers.9.mlp.experts.54.gate_proj", "model.layers.9.mlp.experts.55.gate_proj", "model.layers.9.mlp.experts.56.gate_proj", "model.layers.9.mlp.experts.57.gate_proj", "model.layers.9.mlp.experts.58.gate_proj", "model.layers.9.mlp.experts.59.gate_proj", "model.layers.9.mlp.experts.60.gate_proj", "model.layers.9.mlp.experts.61.gate_proj", "model.layers.9.mlp.experts.62.gate_proj", "model.layers.9.mlp.experts.63.gate_proj", "model.layers.9.mlp.experts.64.gate_proj", "model.layers.9.mlp.experts.65.gate_proj", "model.layers.9.mlp.experts.66.gate_proj", "model.layers.9.mlp.experts.67.gate_proj", "model.layers.9.mlp.experts.68.gate_proj", "model.layers.9.mlp.experts.69.gate_proj", "model.layers.9.mlp.experts.70.gate_proj", "model.layers.9.mlp.experts.71.gate_proj", "model.layers.9.mlp.experts.72.gate_proj", "model.layers.9.mlp.experts.73.gate_proj", "model.layers.9.mlp.experts.74.gate_proj", "model.layers.9.mlp.experts.75.gate_proj", "model.layers.9.mlp.experts.76.gate_proj", "model.layers.9.mlp.experts.77.gate_proj", "model.layers.9.mlp.experts.78.gate_proj", "model.layers.9.mlp.experts.79.gate_proj", "model.layers.9.mlp.experts.80.gate_proj", "model.layers.9.mlp.experts.81.gate_proj", "model.layers.9.mlp.experts.82.gate_proj", "model.layers.9.mlp.experts.83.gate_proj", "model.layers.9.mlp.experts.84.gate_proj", "model.layers.9.mlp.experts.85.gate_proj", "model.layers.9.mlp.experts.86.gate_proj", "model.layers.9.mlp.experts.87.gate_proj", "model.layers.9.mlp.experts.88.gate_proj", "model.layers.9.mlp.experts.89.gate_proj", "model.layers.9.mlp.experts.90.gate_proj", "model.layers.9.mlp.experts.91.gate_proj", "model.layers.9.mlp.experts.92.gate_proj", "model.layers.9.mlp.experts.93.gate_proj", "model.layers.9.mlp.experts.94.gate_proj", "model.layers.9.mlp.experts.95.gate_proj", "model.layers.9.mlp.experts.96.gate_proj", "model.layers.9.mlp.experts.97.gate_proj", "model.layers.9.mlp.experts.98.gate_proj", "model.layers.9.mlp.experts.99.gate_proj", "model.layers.9.mlp.experts.100.gate_proj", "model.layers.9.mlp.experts.101.gate_proj", "model.layers.9.mlp.experts.102.gate_proj", "model.layers.9.mlp.experts.103.gate_proj", "model.layers.9.mlp.experts.104.gate_proj", "model.layers.9.mlp.experts.105.gate_proj", "model.layers.9.mlp.experts.106.gate_proj", "model.layers.9.mlp.experts.107.gate_proj", "model.layers.9.mlp.experts.108.gate_proj", "model.layers.9.mlp.experts.109.gate_proj", "model.layers.9.mlp.experts.110.gate_proj", "model.layers.9.mlp.experts.111.gate_proj", "model.layers.9.mlp.experts.112.gate_proj", "model.layers.9.mlp.experts.113.gate_proj", "model.layers.9.mlp.experts.114.gate_proj", "model.layers.9.mlp.experts.115.gate_proj", "model.layers.9.mlp.experts.116.gate_proj", "model.layers.9.mlp.experts.117.gate_proj", "model.layers.9.mlp.experts.118.gate_proj", "model.layers.9.mlp.experts.119.gate_proj", "model.layers.9.mlp.experts.120.gate_proj", "model.layers.9.mlp.experts.121.gate_proj", "model.layers.9.mlp.experts.122.gate_proj", "model.layers.9.mlp.experts.123.gate_proj", "model.layers.9.mlp.experts.124.gate_proj", "model.layers.9.mlp.experts.125.gate_proj", "model.layers.9.mlp.experts.126.gate_proj", "model.layers.9.mlp.experts.127.gate_proj", "model.layers.9.mlp.experts.128.gate_proj", "model.layers.9.mlp.experts.129.gate_proj", "model.layers.9.mlp.experts.130.gate_proj", "model.layers.9.mlp.experts.131.gate_proj", "model.layers.9.mlp.experts.132.gate_proj", "model.layers.9.mlp.experts.133.gate_proj", "model.layers.9.mlp.experts.134.gate_proj", "model.layers.9.mlp.experts.135.gate_proj", "model.layers.9.mlp.experts.136.gate_proj", "model.layers.9.mlp.experts.137.gate_proj", "model.layers.9.mlp.experts.138.gate_proj", "model.layers.9.mlp.experts.139.gate_proj", "model.layers.9.mlp.experts.140.gate_proj", "model.layers.9.mlp.experts.141.gate_proj", "model.layers.9.mlp.experts.142.gate_proj", "model.layers.9.mlp.experts.143.gate_proj", "model.layers.9.mlp.experts.144.gate_proj", "model.layers.9.mlp.experts.145.gate_proj", "model.layers.9.mlp.experts.146.gate_proj", "model.layers.9.mlp.experts.147.gate_proj", "model.layers.9.mlp.experts.148.gate_proj", "model.layers.9.mlp.experts.149.gate_proj", "model.layers.9.mlp.experts.150.gate_proj", "model.layers.9.mlp.experts.151.gate_proj", "model.layers.9.mlp.experts.152.gate_proj", "model.layers.9.mlp.experts.153.gate_proj", "model.layers.9.mlp.experts.154.gate_proj", "model.layers.9.mlp.experts.155.gate_proj", "model.layers.9.mlp.experts.156.gate_proj", "model.layers.9.mlp.experts.157.gate_proj", "model.layers.9.mlp.experts.158.gate_proj", "model.layers.9.mlp.experts.159.gate_proj", "model.layers.9.mlp.experts.0.up_proj", "model.layers.9.mlp.experts.1.up_proj", "model.layers.9.mlp.experts.2.up_proj", "model.layers.9.mlp.experts.3.up_proj", "model.layers.9.mlp.experts.4.up_proj", "model.layers.9.mlp.experts.5.up_proj", "model.layers.9.mlp.experts.6.up_proj", "model.layers.9.mlp.experts.7.up_proj", "model.layers.9.mlp.experts.8.up_proj", "model.layers.9.mlp.experts.9.up_proj", "model.layers.9.mlp.experts.10.up_proj", "model.layers.9.mlp.experts.11.up_proj", "model.layers.9.mlp.experts.12.up_proj", "model.layers.9.mlp.experts.13.up_proj", "model.layers.9.mlp.experts.14.up_proj", "model.layers.9.mlp.experts.15.up_proj", "model.layers.9.mlp.experts.16.up_proj", "model.layers.9.mlp.experts.17.up_proj", "model.layers.9.mlp.experts.18.up_proj", "model.layers.9.mlp.experts.19.up_proj", "model.layers.9.mlp.experts.20.up_proj", "model.layers.9.mlp.experts.21.up_proj", "model.layers.9.mlp.experts.22.up_proj", "model.layers.9.mlp.experts.23.up_proj", "model.layers.9.mlp.experts.24.up_proj", "model.layers.9.mlp.experts.25.up_proj", "model.layers.9.mlp.experts.26.up_proj", "model.layers.9.mlp.experts.27.up_proj", "model.layers.9.mlp.experts.28.up_proj", "model.layers.9.mlp.experts.29.up_proj", "model.layers.9.mlp.experts.30.up_proj", "model.layers.9.mlp.experts.31.up_proj", "model.layers.9.mlp.experts.32.up_proj", "model.layers.9.mlp.experts.33.up_proj", "model.layers.9.mlp.experts.34.up_proj", "model.layers.9.mlp.experts.35.up_proj", "model.layers.9.mlp.experts.36.up_proj", "model.layers.9.mlp.experts.37.up_proj", "model.layers.9.mlp.experts.38.up_proj", "model.layers.9.mlp.experts.39.up_proj", "model.layers.9.mlp.experts.40.up_proj", "model.layers.9.mlp.experts.41.up_proj", "model.layers.9.mlp.experts.42.up_proj", "model.layers.9.mlp.experts.43.up_proj", "model.layers.9.mlp.experts.44.up_proj", "model.layers.9.mlp.experts.45.up_proj", "model.layers.9.mlp.experts.46.up_proj", "model.layers.9.mlp.experts.47.up_proj", "model.layers.9.mlp.experts.48.up_proj", "model.layers.9.mlp.experts.49.up_proj", "model.layers.9.mlp.experts.50.up_proj", "model.layers.9.mlp.experts.51.up_proj", "model.layers.9.mlp.experts.52.up_proj", "model.layers.9.mlp.experts.53.up_proj", "model.layers.9.mlp.experts.54.up_proj", "model.layers.9.mlp.experts.55.up_proj", "model.layers.9.mlp.experts.56.up_proj", "model.layers.9.mlp.experts.57.up_proj", "model.layers.9.mlp.experts.58.up_proj", "model.layers.9.mlp.experts.59.up_proj", "model.layers.9.mlp.experts.60.up_proj", "model.layers.9.mlp.experts.61.up_proj", "model.layers.9.mlp.experts.62.up_proj", "model.layers.9.mlp.experts.63.up_proj", "model.layers.9.mlp.experts.64.up_proj", "model.layers.9.mlp.experts.65.up_proj", "model.layers.9.mlp.experts.66.up_proj", "model.layers.9.mlp.experts.67.up_proj", "model.layers.9.mlp.experts.68.up_proj", "model.layers.9.mlp.experts.69.up_proj", "model.layers.9.mlp.experts.70.up_proj", "model.layers.9.mlp.experts.71.up_proj", "model.layers.9.mlp.experts.72.up_proj", "model.layers.9.mlp.experts.73.up_proj", "model.layers.9.mlp.experts.74.up_proj", "model.layers.9.mlp.experts.75.up_proj", "model.layers.9.mlp.experts.76.up_proj", "model.layers.9.mlp.experts.77.up_proj", "model.layers.9.mlp.experts.78.up_proj", "model.layers.9.mlp.experts.79.up_proj", "model.layers.9.mlp.experts.80.up_proj", "model.layers.9.mlp.experts.81.up_proj", "model.layers.9.mlp.experts.82.up_proj", "model.layers.9.mlp.experts.83.up_proj", "model.layers.9.mlp.experts.84.up_proj", "model.layers.9.mlp.experts.85.up_proj", "model.layers.9.mlp.experts.86.up_proj", "model.layers.9.mlp.experts.87.up_proj", "model.layers.9.mlp.experts.88.up_proj", "model.layers.9.mlp.experts.89.up_proj", "model.layers.9.mlp.experts.90.up_proj", "model.layers.9.mlp.experts.91.up_proj", "model.layers.9.mlp.experts.92.up_proj", "model.layers.9.mlp.experts.93.up_proj", "model.layers.9.mlp.experts.94.up_proj", "model.layers.9.mlp.experts.95.up_proj", "model.layers.9.mlp.experts.96.up_proj", "model.layers.9.mlp.experts.97.up_proj", "model.layers.9.mlp.experts.98.up_proj", "model.layers.9.mlp.experts.99.up_proj", "model.layers.9.mlp.experts.100.up_proj", "model.layers.9.mlp.experts.101.up_proj", "model.layers.9.mlp.experts.102.up_proj", "model.layers.9.mlp.experts.103.up_proj", "model.layers.9.mlp.experts.104.up_proj", "model.layers.9.mlp.experts.105.up_proj", "model.layers.9.mlp.experts.106.up_proj", "model.layers.9.mlp.experts.107.up_proj", "model.layers.9.mlp.experts.108.up_proj", "model.layers.9.mlp.experts.109.up_proj", "model.layers.9.mlp.experts.110.up_proj", "model.layers.9.mlp.experts.111.up_proj", "model.layers.9.mlp.experts.112.up_proj", "model.layers.9.mlp.experts.113.up_proj", "model.layers.9.mlp.experts.114.up_proj", "model.layers.9.mlp.experts.115.up_proj", "model.layers.9.mlp.experts.116.up_proj", "model.layers.9.mlp.experts.117.up_proj", "model.layers.9.mlp.experts.118.up_proj", "model.layers.9.mlp.experts.119.up_proj", "model.layers.9.mlp.experts.120.up_proj", "model.layers.9.mlp.experts.121.up_proj", "model.layers.9.mlp.experts.122.up_proj", "model.layers.9.mlp.experts.123.up_proj", "model.layers.9.mlp.experts.124.up_proj", "model.layers.9.mlp.experts.125.up_proj", "model.layers.9.mlp.experts.126.up_proj", "model.layers.9.mlp.experts.127.up_proj", "model.layers.9.mlp.experts.128.up_proj", "model.layers.9.mlp.experts.129.up_proj", "model.layers.9.mlp.experts.130.up_proj", "model.layers.9.mlp.experts.131.up_proj", "model.layers.9.mlp.experts.132.up_proj", "model.layers.9.mlp.experts.133.up_proj", "model.layers.9.mlp.experts.134.up_proj", "model.layers.9.mlp.experts.135.up_proj", "model.layers.9.mlp.experts.136.up_proj", "model.layers.9.mlp.experts.137.up_proj", "model.layers.9.mlp.experts.138.up_proj", "model.layers.9.mlp.experts.139.up_proj", "model.layers.9.mlp.experts.140.up_proj", "model.layers.9.mlp.experts.141.up_proj", "model.layers.9.mlp.experts.142.up_proj", "model.layers.9.mlp.experts.143.up_proj", "model.layers.9.mlp.experts.144.up_proj", "model.layers.9.mlp.experts.145.up_proj", "model.layers.9.mlp.experts.146.up_proj", "model.layers.9.mlp.experts.147.up_proj", "model.layers.9.mlp.experts.148.up_proj", "model.layers.9.mlp.experts.149.up_proj", "model.layers.9.mlp.experts.150.up_proj", "model.layers.9.mlp.experts.151.up_proj", "model.layers.9.mlp.experts.152.up_proj", "model.layers.9.mlp.experts.153.up_proj", "model.layers.9.mlp.experts.154.up_proj", "model.layers.9.mlp.experts.155.up_proj", "model.layers.9.mlp.experts.156.up_proj", "model.layers.9.mlp.experts.157.up_proj", "model.layers.9.mlp.experts.158.up_proj", "model.layers.9.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -2.4804941494949184e-05, "dbits": 5033164800 } ] }, { "idx": 56, "layers": [ "model.layers.9.mlp.experts.0.down_proj", "model.layers.9.mlp.experts.1.down_proj", "model.layers.9.mlp.experts.2.down_proj", "model.layers.9.mlp.experts.3.down_proj", "model.layers.9.mlp.experts.4.down_proj", "model.layers.9.mlp.experts.5.down_proj", "model.layers.9.mlp.experts.6.down_proj", "model.layers.9.mlp.experts.7.down_proj", "model.layers.9.mlp.experts.8.down_proj", "model.layers.9.mlp.experts.9.down_proj", "model.layers.9.mlp.experts.10.down_proj", "model.layers.9.mlp.experts.11.down_proj", "model.layers.9.mlp.experts.12.down_proj", "model.layers.9.mlp.experts.13.down_proj", "model.layers.9.mlp.experts.14.down_proj", "model.layers.9.mlp.experts.15.down_proj", "model.layers.9.mlp.experts.16.down_proj", "model.layers.9.mlp.experts.17.down_proj", "model.layers.9.mlp.experts.18.down_proj", "model.layers.9.mlp.experts.19.down_proj", "model.layers.9.mlp.experts.20.down_proj", "model.layers.9.mlp.experts.21.down_proj", "model.layers.9.mlp.experts.22.down_proj", "model.layers.9.mlp.experts.23.down_proj", "model.layers.9.mlp.experts.24.down_proj", "model.layers.9.mlp.experts.25.down_proj", "model.layers.9.mlp.experts.26.down_proj", "model.layers.9.mlp.experts.27.down_proj", "model.layers.9.mlp.experts.28.down_proj", "model.layers.9.mlp.experts.29.down_proj", "model.layers.9.mlp.experts.30.down_proj", "model.layers.9.mlp.experts.31.down_proj", "model.layers.9.mlp.experts.32.down_proj", "model.layers.9.mlp.experts.33.down_proj", "model.layers.9.mlp.experts.34.down_proj", "model.layers.9.mlp.experts.35.down_proj", "model.layers.9.mlp.experts.36.down_proj", "model.layers.9.mlp.experts.37.down_proj", "model.layers.9.mlp.experts.38.down_proj", "model.layers.9.mlp.experts.39.down_proj", "model.layers.9.mlp.experts.40.down_proj", "model.layers.9.mlp.experts.41.down_proj", "model.layers.9.mlp.experts.42.down_proj", "model.layers.9.mlp.experts.43.down_proj", "model.layers.9.mlp.experts.44.down_proj", "model.layers.9.mlp.experts.45.down_proj", "model.layers.9.mlp.experts.46.down_proj", "model.layers.9.mlp.experts.47.down_proj", "model.layers.9.mlp.experts.48.down_proj", "model.layers.9.mlp.experts.49.down_proj", "model.layers.9.mlp.experts.50.down_proj", "model.layers.9.mlp.experts.51.down_proj", "model.layers.9.mlp.experts.52.down_proj", "model.layers.9.mlp.experts.53.down_proj", "model.layers.9.mlp.experts.54.down_proj", "model.layers.9.mlp.experts.55.down_proj", "model.layers.9.mlp.experts.56.down_proj", "model.layers.9.mlp.experts.57.down_proj", "model.layers.9.mlp.experts.58.down_proj", "model.layers.9.mlp.experts.59.down_proj", "model.layers.9.mlp.experts.60.down_proj", "model.layers.9.mlp.experts.61.down_proj", "model.layers.9.mlp.experts.62.down_proj", "model.layers.9.mlp.experts.63.down_proj", "model.layers.9.mlp.experts.64.down_proj", "model.layers.9.mlp.experts.65.down_proj", "model.layers.9.mlp.experts.66.down_proj", "model.layers.9.mlp.experts.67.down_proj", "model.layers.9.mlp.experts.68.down_proj", "model.layers.9.mlp.experts.69.down_proj", "model.layers.9.mlp.experts.70.down_proj", "model.layers.9.mlp.experts.71.down_proj", "model.layers.9.mlp.experts.72.down_proj", "model.layers.9.mlp.experts.73.down_proj", "model.layers.9.mlp.experts.74.down_proj", "model.layers.9.mlp.experts.75.down_proj", "model.layers.9.mlp.experts.76.down_proj", "model.layers.9.mlp.experts.77.down_proj", "model.layers.9.mlp.experts.78.down_proj", "model.layers.9.mlp.experts.79.down_proj", "model.layers.9.mlp.experts.80.down_proj", "model.layers.9.mlp.experts.81.down_proj", "model.layers.9.mlp.experts.82.down_proj", "model.layers.9.mlp.experts.83.down_proj", "model.layers.9.mlp.experts.84.down_proj", "model.layers.9.mlp.experts.85.down_proj", "model.layers.9.mlp.experts.86.down_proj", "model.layers.9.mlp.experts.87.down_proj", "model.layers.9.mlp.experts.88.down_proj", "model.layers.9.mlp.experts.89.down_proj", "model.layers.9.mlp.experts.90.down_proj", "model.layers.9.mlp.experts.91.down_proj", "model.layers.9.mlp.experts.92.down_proj", "model.layers.9.mlp.experts.93.down_proj", "model.layers.9.mlp.experts.94.down_proj", "model.layers.9.mlp.experts.95.down_proj", "model.layers.9.mlp.experts.96.down_proj", "model.layers.9.mlp.experts.97.down_proj", "model.layers.9.mlp.experts.98.down_proj", "model.layers.9.mlp.experts.99.down_proj", "model.layers.9.mlp.experts.100.down_proj", "model.layers.9.mlp.experts.101.down_proj", "model.layers.9.mlp.experts.102.down_proj", "model.layers.9.mlp.experts.103.down_proj", "model.layers.9.mlp.experts.104.down_proj", "model.layers.9.mlp.experts.105.down_proj", "model.layers.9.mlp.experts.106.down_proj", "model.layers.9.mlp.experts.107.down_proj", "model.layers.9.mlp.experts.108.down_proj", "model.layers.9.mlp.experts.109.down_proj", "model.layers.9.mlp.experts.110.down_proj", "model.layers.9.mlp.experts.111.down_proj", "model.layers.9.mlp.experts.112.down_proj", "model.layers.9.mlp.experts.113.down_proj", "model.layers.9.mlp.experts.114.down_proj", "model.layers.9.mlp.experts.115.down_proj", "model.layers.9.mlp.experts.116.down_proj", "model.layers.9.mlp.experts.117.down_proj", "model.layers.9.mlp.experts.118.down_proj", "model.layers.9.mlp.experts.119.down_proj", "model.layers.9.mlp.experts.120.down_proj", "model.layers.9.mlp.experts.121.down_proj", "model.layers.9.mlp.experts.122.down_proj", "model.layers.9.mlp.experts.123.down_proj", "model.layers.9.mlp.experts.124.down_proj", "model.layers.9.mlp.experts.125.down_proj", "model.layers.9.mlp.experts.126.down_proj", "model.layers.9.mlp.experts.127.down_proj", "model.layers.9.mlp.experts.128.down_proj", "model.layers.9.mlp.experts.129.down_proj", "model.layers.9.mlp.experts.130.down_proj", "model.layers.9.mlp.experts.131.down_proj", "model.layers.9.mlp.experts.132.down_proj", "model.layers.9.mlp.experts.133.down_proj", "model.layers.9.mlp.experts.134.down_proj", "model.layers.9.mlp.experts.135.down_proj", "model.layers.9.mlp.experts.136.down_proj", "model.layers.9.mlp.experts.137.down_proj", "model.layers.9.mlp.experts.138.down_proj", "model.layers.9.mlp.experts.139.down_proj", "model.layers.9.mlp.experts.140.down_proj", "model.layers.9.mlp.experts.141.down_proj", "model.layers.9.mlp.experts.142.down_proj", "model.layers.9.mlp.experts.143.down_proj", "model.layers.9.mlp.experts.144.down_proj", "model.layers.9.mlp.experts.145.down_proj", "model.layers.9.mlp.experts.146.down_proj", "model.layers.9.mlp.experts.147.down_proj", "model.layers.9.mlp.experts.148.down_proj", "model.layers.9.mlp.experts.149.down_proj", "model.layers.9.mlp.experts.150.down_proj", "model.layers.9.mlp.experts.151.down_proj", "model.layers.9.mlp.experts.152.down_proj", "model.layers.9.mlp.experts.153.down_proj", "model.layers.9.mlp.experts.154.down_proj", "model.layers.9.mlp.experts.155.down_proj", "model.layers.9.mlp.experts.156.down_proj", "model.layers.9.mlp.experts.157.down_proj", "model.layers.9.mlp.experts.158.down_proj", "model.layers.9.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -7.004625513218387e-06, "dbits": 2516582400 } ] }, { "idx": 57, "layers": [ "model.layers.10.self_attn.q_proj" ], "candidates": [ { "dkld": 2.0718001178465827e-05, "dbits": 125829120 } ] }, { "idx": 58, "layers": [ "model.layers.10.self_attn.k_proj", "model.layers.10.self_attn.v_proj" ], "candidates": [ { "dkld": 9.275702177547005e-06, "dbits": 20971520 } ] }, { "idx": 59, "layers": [ "model.layers.10.self_attn.o_proj" ], "candidates": [ { "dkld": 2.6184608577750536e-05, "dbits": 125829120 } ] }, { "idx": 60, "layers": [ "model.layers.10.mlp.shared_experts.gate_proj", "model.layers.10.mlp.shared_experts.up_proj", "model.layers.10.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -7.86702730692923e-06, "dbits": 47185920 } ] }, { "idx": 61, "layers": [ "model.layers.10.mlp.experts.0.gate_proj", "model.layers.10.mlp.experts.1.gate_proj", "model.layers.10.mlp.experts.2.gate_proj", "model.layers.10.mlp.experts.3.gate_proj", "model.layers.10.mlp.experts.4.gate_proj", "model.layers.10.mlp.experts.5.gate_proj", "model.layers.10.mlp.experts.6.gate_proj", "model.layers.10.mlp.experts.7.gate_proj", "model.layers.10.mlp.experts.8.gate_proj", "model.layers.10.mlp.experts.9.gate_proj", "model.layers.10.mlp.experts.10.gate_proj", "model.layers.10.mlp.experts.11.gate_proj", "model.layers.10.mlp.experts.12.gate_proj", "model.layers.10.mlp.experts.13.gate_proj", "model.layers.10.mlp.experts.14.gate_proj", "model.layers.10.mlp.experts.15.gate_proj", "model.layers.10.mlp.experts.16.gate_proj", "model.layers.10.mlp.experts.17.gate_proj", "model.layers.10.mlp.experts.18.gate_proj", "model.layers.10.mlp.experts.19.gate_proj", "model.layers.10.mlp.experts.20.gate_proj", "model.layers.10.mlp.experts.21.gate_proj", "model.layers.10.mlp.experts.22.gate_proj", "model.layers.10.mlp.experts.23.gate_proj", "model.layers.10.mlp.experts.24.gate_proj", "model.layers.10.mlp.experts.25.gate_proj", "model.layers.10.mlp.experts.26.gate_proj", "model.layers.10.mlp.experts.27.gate_proj", "model.layers.10.mlp.experts.28.gate_proj", "model.layers.10.mlp.experts.29.gate_proj", "model.layers.10.mlp.experts.30.gate_proj", "model.layers.10.mlp.experts.31.gate_proj", "model.layers.10.mlp.experts.32.gate_proj", "model.layers.10.mlp.experts.33.gate_proj", "model.layers.10.mlp.experts.34.gate_proj", "model.layers.10.mlp.experts.35.gate_proj", "model.layers.10.mlp.experts.36.gate_proj", "model.layers.10.mlp.experts.37.gate_proj", "model.layers.10.mlp.experts.38.gate_proj", "model.layers.10.mlp.experts.39.gate_proj", "model.layers.10.mlp.experts.40.gate_proj", "model.layers.10.mlp.experts.41.gate_proj", "model.layers.10.mlp.experts.42.gate_proj", "model.layers.10.mlp.experts.43.gate_proj", "model.layers.10.mlp.experts.44.gate_proj", "model.layers.10.mlp.experts.45.gate_proj", "model.layers.10.mlp.experts.46.gate_proj", "model.layers.10.mlp.experts.47.gate_proj", "model.layers.10.mlp.experts.48.gate_proj", "model.layers.10.mlp.experts.49.gate_proj", "model.layers.10.mlp.experts.50.gate_proj", "model.layers.10.mlp.experts.51.gate_proj", "model.layers.10.mlp.experts.52.gate_proj", "model.layers.10.mlp.experts.53.gate_proj", "model.layers.10.mlp.experts.54.gate_proj", "model.layers.10.mlp.experts.55.gate_proj", "model.layers.10.mlp.experts.56.gate_proj", "model.layers.10.mlp.experts.57.gate_proj", "model.layers.10.mlp.experts.58.gate_proj", "model.layers.10.mlp.experts.59.gate_proj", "model.layers.10.mlp.experts.60.gate_proj", "model.layers.10.mlp.experts.61.gate_proj", "model.layers.10.mlp.experts.62.gate_proj", "model.layers.10.mlp.experts.63.gate_proj", "model.layers.10.mlp.experts.64.gate_proj", "model.layers.10.mlp.experts.65.gate_proj", "model.layers.10.mlp.experts.66.gate_proj", "model.layers.10.mlp.experts.67.gate_proj", "model.layers.10.mlp.experts.68.gate_proj", "model.layers.10.mlp.experts.69.gate_proj", "model.layers.10.mlp.experts.70.gate_proj", "model.layers.10.mlp.experts.71.gate_proj", "model.layers.10.mlp.experts.72.gate_proj", "model.layers.10.mlp.experts.73.gate_proj", "model.layers.10.mlp.experts.74.gate_proj", "model.layers.10.mlp.experts.75.gate_proj", "model.layers.10.mlp.experts.76.gate_proj", "model.layers.10.mlp.experts.77.gate_proj", "model.layers.10.mlp.experts.78.gate_proj", "model.layers.10.mlp.experts.79.gate_proj", "model.layers.10.mlp.experts.80.gate_proj", "model.layers.10.mlp.experts.81.gate_proj", "model.layers.10.mlp.experts.82.gate_proj", "model.layers.10.mlp.experts.83.gate_proj", "model.layers.10.mlp.experts.84.gate_proj", "model.layers.10.mlp.experts.85.gate_proj", "model.layers.10.mlp.experts.86.gate_proj", "model.layers.10.mlp.experts.87.gate_proj", "model.layers.10.mlp.experts.88.gate_proj", "model.layers.10.mlp.experts.89.gate_proj", "model.layers.10.mlp.experts.90.gate_proj", "model.layers.10.mlp.experts.91.gate_proj", "model.layers.10.mlp.experts.92.gate_proj", "model.layers.10.mlp.experts.93.gate_proj", "model.layers.10.mlp.experts.94.gate_proj", "model.layers.10.mlp.experts.95.gate_proj", "model.layers.10.mlp.experts.96.gate_proj", "model.layers.10.mlp.experts.97.gate_proj", "model.layers.10.mlp.experts.98.gate_proj", "model.layers.10.mlp.experts.99.gate_proj", "model.layers.10.mlp.experts.100.gate_proj", "model.layers.10.mlp.experts.101.gate_proj", "model.layers.10.mlp.experts.102.gate_proj", "model.layers.10.mlp.experts.103.gate_proj", "model.layers.10.mlp.experts.104.gate_proj", "model.layers.10.mlp.experts.105.gate_proj", "model.layers.10.mlp.experts.106.gate_proj", "model.layers.10.mlp.experts.107.gate_proj", "model.layers.10.mlp.experts.108.gate_proj", "model.layers.10.mlp.experts.109.gate_proj", "model.layers.10.mlp.experts.110.gate_proj", "model.layers.10.mlp.experts.111.gate_proj", "model.layers.10.mlp.experts.112.gate_proj", "model.layers.10.mlp.experts.113.gate_proj", "model.layers.10.mlp.experts.114.gate_proj", "model.layers.10.mlp.experts.115.gate_proj", "model.layers.10.mlp.experts.116.gate_proj", "model.layers.10.mlp.experts.117.gate_proj", "model.layers.10.mlp.experts.118.gate_proj", "model.layers.10.mlp.experts.119.gate_proj", "model.layers.10.mlp.experts.120.gate_proj", "model.layers.10.mlp.experts.121.gate_proj", "model.layers.10.mlp.experts.122.gate_proj", "model.layers.10.mlp.experts.123.gate_proj", "model.layers.10.mlp.experts.124.gate_proj", "model.layers.10.mlp.experts.125.gate_proj", "model.layers.10.mlp.experts.126.gate_proj", "model.layers.10.mlp.experts.127.gate_proj", "model.layers.10.mlp.experts.128.gate_proj", "model.layers.10.mlp.experts.129.gate_proj", "model.layers.10.mlp.experts.130.gate_proj", "model.layers.10.mlp.experts.131.gate_proj", "model.layers.10.mlp.experts.132.gate_proj", "model.layers.10.mlp.experts.133.gate_proj", "model.layers.10.mlp.experts.134.gate_proj", "model.layers.10.mlp.experts.135.gate_proj", "model.layers.10.mlp.experts.136.gate_proj", "model.layers.10.mlp.experts.137.gate_proj", "model.layers.10.mlp.experts.138.gate_proj", "model.layers.10.mlp.experts.139.gate_proj", "model.layers.10.mlp.experts.140.gate_proj", "model.layers.10.mlp.experts.141.gate_proj", "model.layers.10.mlp.experts.142.gate_proj", "model.layers.10.mlp.experts.143.gate_proj", "model.layers.10.mlp.experts.144.gate_proj", "model.layers.10.mlp.experts.145.gate_proj", "model.layers.10.mlp.experts.146.gate_proj", "model.layers.10.mlp.experts.147.gate_proj", "model.layers.10.mlp.experts.148.gate_proj", "model.layers.10.mlp.experts.149.gate_proj", "model.layers.10.mlp.experts.150.gate_proj", "model.layers.10.mlp.experts.151.gate_proj", "model.layers.10.mlp.experts.152.gate_proj", "model.layers.10.mlp.experts.153.gate_proj", "model.layers.10.mlp.experts.154.gate_proj", "model.layers.10.mlp.experts.155.gate_proj", "model.layers.10.mlp.experts.156.gate_proj", "model.layers.10.mlp.experts.157.gate_proj", "model.layers.10.mlp.experts.158.gate_proj", "model.layers.10.mlp.experts.159.gate_proj", "model.layers.10.mlp.experts.0.up_proj", "model.layers.10.mlp.experts.1.up_proj", "model.layers.10.mlp.experts.2.up_proj", "model.layers.10.mlp.experts.3.up_proj", "model.layers.10.mlp.experts.4.up_proj", "model.layers.10.mlp.experts.5.up_proj", "model.layers.10.mlp.experts.6.up_proj", "model.layers.10.mlp.experts.7.up_proj", "model.layers.10.mlp.experts.8.up_proj", "model.layers.10.mlp.experts.9.up_proj", "model.layers.10.mlp.experts.10.up_proj", "model.layers.10.mlp.experts.11.up_proj", "model.layers.10.mlp.experts.12.up_proj", "model.layers.10.mlp.experts.13.up_proj", "model.layers.10.mlp.experts.14.up_proj", "model.layers.10.mlp.experts.15.up_proj", "model.layers.10.mlp.experts.16.up_proj", "model.layers.10.mlp.experts.17.up_proj", "model.layers.10.mlp.experts.18.up_proj", "model.layers.10.mlp.experts.19.up_proj", "model.layers.10.mlp.experts.20.up_proj", "model.layers.10.mlp.experts.21.up_proj", "model.layers.10.mlp.experts.22.up_proj", "model.layers.10.mlp.experts.23.up_proj", "model.layers.10.mlp.experts.24.up_proj", "model.layers.10.mlp.experts.25.up_proj", "model.layers.10.mlp.experts.26.up_proj", "model.layers.10.mlp.experts.27.up_proj", "model.layers.10.mlp.experts.28.up_proj", "model.layers.10.mlp.experts.29.up_proj", "model.layers.10.mlp.experts.30.up_proj", "model.layers.10.mlp.experts.31.up_proj", "model.layers.10.mlp.experts.32.up_proj", "model.layers.10.mlp.experts.33.up_proj", "model.layers.10.mlp.experts.34.up_proj", "model.layers.10.mlp.experts.35.up_proj", "model.layers.10.mlp.experts.36.up_proj", "model.layers.10.mlp.experts.37.up_proj", "model.layers.10.mlp.experts.38.up_proj", "model.layers.10.mlp.experts.39.up_proj", "model.layers.10.mlp.experts.40.up_proj", "model.layers.10.mlp.experts.41.up_proj", "model.layers.10.mlp.experts.42.up_proj", "model.layers.10.mlp.experts.43.up_proj", "model.layers.10.mlp.experts.44.up_proj", "model.layers.10.mlp.experts.45.up_proj", "model.layers.10.mlp.experts.46.up_proj", "model.layers.10.mlp.experts.47.up_proj", "model.layers.10.mlp.experts.48.up_proj", "model.layers.10.mlp.experts.49.up_proj", "model.layers.10.mlp.experts.50.up_proj", "model.layers.10.mlp.experts.51.up_proj", "model.layers.10.mlp.experts.52.up_proj", "model.layers.10.mlp.experts.53.up_proj", "model.layers.10.mlp.experts.54.up_proj", "model.layers.10.mlp.experts.55.up_proj", "model.layers.10.mlp.experts.56.up_proj", "model.layers.10.mlp.experts.57.up_proj", "model.layers.10.mlp.experts.58.up_proj", "model.layers.10.mlp.experts.59.up_proj", "model.layers.10.mlp.experts.60.up_proj", "model.layers.10.mlp.experts.61.up_proj", "model.layers.10.mlp.experts.62.up_proj", "model.layers.10.mlp.experts.63.up_proj", "model.layers.10.mlp.experts.64.up_proj", "model.layers.10.mlp.experts.65.up_proj", "model.layers.10.mlp.experts.66.up_proj", "model.layers.10.mlp.experts.67.up_proj", "model.layers.10.mlp.experts.68.up_proj", "model.layers.10.mlp.experts.69.up_proj", "model.layers.10.mlp.experts.70.up_proj", "model.layers.10.mlp.experts.71.up_proj", "model.layers.10.mlp.experts.72.up_proj", "model.layers.10.mlp.experts.73.up_proj", "model.layers.10.mlp.experts.74.up_proj", "model.layers.10.mlp.experts.75.up_proj", "model.layers.10.mlp.experts.76.up_proj", "model.layers.10.mlp.experts.77.up_proj", "model.layers.10.mlp.experts.78.up_proj", "model.layers.10.mlp.experts.79.up_proj", "model.layers.10.mlp.experts.80.up_proj", "model.layers.10.mlp.experts.81.up_proj", "model.layers.10.mlp.experts.82.up_proj", "model.layers.10.mlp.experts.83.up_proj", "model.layers.10.mlp.experts.84.up_proj", "model.layers.10.mlp.experts.85.up_proj", "model.layers.10.mlp.experts.86.up_proj", "model.layers.10.mlp.experts.87.up_proj", "model.layers.10.mlp.experts.88.up_proj", "model.layers.10.mlp.experts.89.up_proj", "model.layers.10.mlp.experts.90.up_proj", "model.layers.10.mlp.experts.91.up_proj", "model.layers.10.mlp.experts.92.up_proj", "model.layers.10.mlp.experts.93.up_proj", "model.layers.10.mlp.experts.94.up_proj", "model.layers.10.mlp.experts.95.up_proj", "model.layers.10.mlp.experts.96.up_proj", "model.layers.10.mlp.experts.97.up_proj", "model.layers.10.mlp.experts.98.up_proj", "model.layers.10.mlp.experts.99.up_proj", "model.layers.10.mlp.experts.100.up_proj", "model.layers.10.mlp.experts.101.up_proj", "model.layers.10.mlp.experts.102.up_proj", "model.layers.10.mlp.experts.103.up_proj", "model.layers.10.mlp.experts.104.up_proj", "model.layers.10.mlp.experts.105.up_proj", "model.layers.10.mlp.experts.106.up_proj", "model.layers.10.mlp.experts.107.up_proj", "model.layers.10.mlp.experts.108.up_proj", "model.layers.10.mlp.experts.109.up_proj", "model.layers.10.mlp.experts.110.up_proj", "model.layers.10.mlp.experts.111.up_proj", "model.layers.10.mlp.experts.112.up_proj", "model.layers.10.mlp.experts.113.up_proj", "model.layers.10.mlp.experts.114.up_proj", "model.layers.10.mlp.experts.115.up_proj", "model.layers.10.mlp.experts.116.up_proj", "model.layers.10.mlp.experts.117.up_proj", "model.layers.10.mlp.experts.118.up_proj", "model.layers.10.mlp.experts.119.up_proj", "model.layers.10.mlp.experts.120.up_proj", "model.layers.10.mlp.experts.121.up_proj", "model.layers.10.mlp.experts.122.up_proj", "model.layers.10.mlp.experts.123.up_proj", "model.layers.10.mlp.experts.124.up_proj", "model.layers.10.mlp.experts.125.up_proj", "model.layers.10.mlp.experts.126.up_proj", "model.layers.10.mlp.experts.127.up_proj", "model.layers.10.mlp.experts.128.up_proj", "model.layers.10.mlp.experts.129.up_proj", "model.layers.10.mlp.experts.130.up_proj", "model.layers.10.mlp.experts.131.up_proj", "model.layers.10.mlp.experts.132.up_proj", "model.layers.10.mlp.experts.133.up_proj", "model.layers.10.mlp.experts.134.up_proj", "model.layers.10.mlp.experts.135.up_proj", "model.layers.10.mlp.experts.136.up_proj", "model.layers.10.mlp.experts.137.up_proj", "model.layers.10.mlp.experts.138.up_proj", "model.layers.10.mlp.experts.139.up_proj", "model.layers.10.mlp.experts.140.up_proj", "model.layers.10.mlp.experts.141.up_proj", "model.layers.10.mlp.experts.142.up_proj", "model.layers.10.mlp.experts.143.up_proj", "model.layers.10.mlp.experts.144.up_proj", "model.layers.10.mlp.experts.145.up_proj", "model.layers.10.mlp.experts.146.up_proj", "model.layers.10.mlp.experts.147.up_proj", "model.layers.10.mlp.experts.148.up_proj", "model.layers.10.mlp.experts.149.up_proj", "model.layers.10.mlp.experts.150.up_proj", "model.layers.10.mlp.experts.151.up_proj", "model.layers.10.mlp.experts.152.up_proj", "model.layers.10.mlp.experts.153.up_proj", "model.layers.10.mlp.experts.154.up_proj", "model.layers.10.mlp.experts.155.up_proj", "model.layers.10.mlp.experts.156.up_proj", "model.layers.10.mlp.experts.157.up_proj", "model.layers.10.mlp.experts.158.up_proj", "model.layers.10.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.2200401397422e-05, "dbits": 5033164800 } ] }, { "idx": 62, "layers": [ "model.layers.10.mlp.experts.0.down_proj", "model.layers.10.mlp.experts.1.down_proj", "model.layers.10.mlp.experts.2.down_proj", "model.layers.10.mlp.experts.3.down_proj", "model.layers.10.mlp.experts.4.down_proj", "model.layers.10.mlp.experts.5.down_proj", "model.layers.10.mlp.experts.6.down_proj", "model.layers.10.mlp.experts.7.down_proj", "model.layers.10.mlp.experts.8.down_proj", "model.layers.10.mlp.experts.9.down_proj", "model.layers.10.mlp.experts.10.down_proj", "model.layers.10.mlp.experts.11.down_proj", "model.layers.10.mlp.experts.12.down_proj", "model.layers.10.mlp.experts.13.down_proj", "model.layers.10.mlp.experts.14.down_proj", "model.layers.10.mlp.experts.15.down_proj", "model.layers.10.mlp.experts.16.down_proj", "model.layers.10.mlp.experts.17.down_proj", "model.layers.10.mlp.experts.18.down_proj", "model.layers.10.mlp.experts.19.down_proj", "model.layers.10.mlp.experts.20.down_proj", "model.layers.10.mlp.experts.21.down_proj", "model.layers.10.mlp.experts.22.down_proj", "model.layers.10.mlp.experts.23.down_proj", "model.layers.10.mlp.experts.24.down_proj", "model.layers.10.mlp.experts.25.down_proj", "model.layers.10.mlp.experts.26.down_proj", "model.layers.10.mlp.experts.27.down_proj", "model.layers.10.mlp.experts.28.down_proj", "model.layers.10.mlp.experts.29.down_proj", "model.layers.10.mlp.experts.30.down_proj", "model.layers.10.mlp.experts.31.down_proj", "model.layers.10.mlp.experts.32.down_proj", "model.layers.10.mlp.experts.33.down_proj", "model.layers.10.mlp.experts.34.down_proj", "model.layers.10.mlp.experts.35.down_proj", "model.layers.10.mlp.experts.36.down_proj", "model.layers.10.mlp.experts.37.down_proj", "model.layers.10.mlp.experts.38.down_proj", "model.layers.10.mlp.experts.39.down_proj", "model.layers.10.mlp.experts.40.down_proj", "model.layers.10.mlp.experts.41.down_proj", "model.layers.10.mlp.experts.42.down_proj", "model.layers.10.mlp.experts.43.down_proj", "model.layers.10.mlp.experts.44.down_proj", "model.layers.10.mlp.experts.45.down_proj", "model.layers.10.mlp.experts.46.down_proj", "model.layers.10.mlp.experts.47.down_proj", "model.layers.10.mlp.experts.48.down_proj", "model.layers.10.mlp.experts.49.down_proj", "model.layers.10.mlp.experts.50.down_proj", "model.layers.10.mlp.experts.51.down_proj", "model.layers.10.mlp.experts.52.down_proj", "model.layers.10.mlp.experts.53.down_proj", "model.layers.10.mlp.experts.54.down_proj", "model.layers.10.mlp.experts.55.down_proj", "model.layers.10.mlp.experts.56.down_proj", "model.layers.10.mlp.experts.57.down_proj", "model.layers.10.mlp.experts.58.down_proj", "model.layers.10.mlp.experts.59.down_proj", "model.layers.10.mlp.experts.60.down_proj", "model.layers.10.mlp.experts.61.down_proj", "model.layers.10.mlp.experts.62.down_proj", "model.layers.10.mlp.experts.63.down_proj", "model.layers.10.mlp.experts.64.down_proj", "model.layers.10.mlp.experts.65.down_proj", "model.layers.10.mlp.experts.66.down_proj", "model.layers.10.mlp.experts.67.down_proj", "model.layers.10.mlp.experts.68.down_proj", "model.layers.10.mlp.experts.69.down_proj", "model.layers.10.mlp.experts.70.down_proj", "model.layers.10.mlp.experts.71.down_proj", "model.layers.10.mlp.experts.72.down_proj", "model.layers.10.mlp.experts.73.down_proj", "model.layers.10.mlp.experts.74.down_proj", "model.layers.10.mlp.experts.75.down_proj", "model.layers.10.mlp.experts.76.down_proj", "model.layers.10.mlp.experts.77.down_proj", "model.layers.10.mlp.experts.78.down_proj", "model.layers.10.mlp.experts.79.down_proj", "model.layers.10.mlp.experts.80.down_proj", "model.layers.10.mlp.experts.81.down_proj", "model.layers.10.mlp.experts.82.down_proj", "model.layers.10.mlp.experts.83.down_proj", "model.layers.10.mlp.experts.84.down_proj", "model.layers.10.mlp.experts.85.down_proj", "model.layers.10.mlp.experts.86.down_proj", "model.layers.10.mlp.experts.87.down_proj", "model.layers.10.mlp.experts.88.down_proj", "model.layers.10.mlp.experts.89.down_proj", "model.layers.10.mlp.experts.90.down_proj", "model.layers.10.mlp.experts.91.down_proj", "model.layers.10.mlp.experts.92.down_proj", "model.layers.10.mlp.experts.93.down_proj", "model.layers.10.mlp.experts.94.down_proj", "model.layers.10.mlp.experts.95.down_proj", "model.layers.10.mlp.experts.96.down_proj", "model.layers.10.mlp.experts.97.down_proj", "model.layers.10.mlp.experts.98.down_proj", "model.layers.10.mlp.experts.99.down_proj", "model.layers.10.mlp.experts.100.down_proj", "model.layers.10.mlp.experts.101.down_proj", "model.layers.10.mlp.experts.102.down_proj", "model.layers.10.mlp.experts.103.down_proj", "model.layers.10.mlp.experts.104.down_proj", "model.layers.10.mlp.experts.105.down_proj", "model.layers.10.mlp.experts.106.down_proj", "model.layers.10.mlp.experts.107.down_proj", "model.layers.10.mlp.experts.108.down_proj", "model.layers.10.mlp.experts.109.down_proj", "model.layers.10.mlp.experts.110.down_proj", "model.layers.10.mlp.experts.111.down_proj", "model.layers.10.mlp.experts.112.down_proj", "model.layers.10.mlp.experts.113.down_proj", "model.layers.10.mlp.experts.114.down_proj", "model.layers.10.mlp.experts.115.down_proj", "model.layers.10.mlp.experts.116.down_proj", "model.layers.10.mlp.experts.117.down_proj", "model.layers.10.mlp.experts.118.down_proj", "model.layers.10.mlp.experts.119.down_proj", "model.layers.10.mlp.experts.120.down_proj", "model.layers.10.mlp.experts.121.down_proj", "model.layers.10.mlp.experts.122.down_proj", "model.layers.10.mlp.experts.123.down_proj", "model.layers.10.mlp.experts.124.down_proj", "model.layers.10.mlp.experts.125.down_proj", "model.layers.10.mlp.experts.126.down_proj", "model.layers.10.mlp.experts.127.down_proj", "model.layers.10.mlp.experts.128.down_proj", "model.layers.10.mlp.experts.129.down_proj", "model.layers.10.mlp.experts.130.down_proj", "model.layers.10.mlp.experts.131.down_proj", "model.layers.10.mlp.experts.132.down_proj", "model.layers.10.mlp.experts.133.down_proj", "model.layers.10.mlp.experts.134.down_proj", "model.layers.10.mlp.experts.135.down_proj", "model.layers.10.mlp.experts.136.down_proj", "model.layers.10.mlp.experts.137.down_proj", "model.layers.10.mlp.experts.138.down_proj", "model.layers.10.mlp.experts.139.down_proj", "model.layers.10.mlp.experts.140.down_proj", "model.layers.10.mlp.experts.141.down_proj", "model.layers.10.mlp.experts.142.down_proj", "model.layers.10.mlp.experts.143.down_proj", "model.layers.10.mlp.experts.144.down_proj", "model.layers.10.mlp.experts.145.down_proj", "model.layers.10.mlp.experts.146.down_proj", "model.layers.10.mlp.experts.147.down_proj", "model.layers.10.mlp.experts.148.down_proj", "model.layers.10.mlp.experts.149.down_proj", "model.layers.10.mlp.experts.150.down_proj", "model.layers.10.mlp.experts.151.down_proj", "model.layers.10.mlp.experts.152.down_proj", "model.layers.10.mlp.experts.153.down_proj", "model.layers.10.mlp.experts.154.down_proj", "model.layers.10.mlp.experts.155.down_proj", "model.layers.10.mlp.experts.156.down_proj", "model.layers.10.mlp.experts.157.down_proj", "model.layers.10.mlp.experts.158.down_proj", "model.layers.10.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.3313814997673013e-05, "dbits": 2516582400 } ] }, { "idx": 63, "layers": [ "model.layers.11.self_attn.q_proj" ], "candidates": [ { "dkld": 1.4714285498484969e-05, "dbits": 125829120 } ] }, { "idx": 64, "layers": [ "model.layers.11.self_attn.k_proj", "model.layers.11.self_attn.v_proj" ], "candidates": [ { "dkld": -3.414079837966706e-05, "dbits": 20971520 } ] }, { "idx": 65, "layers": [ "model.layers.11.self_attn.o_proj" ], "candidates": [ { "dkld": -3.1154448515735545e-05, "dbits": 125829120 } ] }, { "idx": 66, "layers": [ "model.layers.11.mlp.shared_experts.gate_proj", "model.layers.11.mlp.shared_experts.up_proj", "model.layers.11.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -5.67640527151525e-05, "dbits": 47185920 } ] }, { "idx": 67, "layers": [ "model.layers.11.mlp.experts.0.gate_proj", "model.layers.11.mlp.experts.1.gate_proj", "model.layers.11.mlp.experts.2.gate_proj", "model.layers.11.mlp.experts.3.gate_proj", "model.layers.11.mlp.experts.4.gate_proj", "model.layers.11.mlp.experts.5.gate_proj", "model.layers.11.mlp.experts.6.gate_proj", "model.layers.11.mlp.experts.7.gate_proj", "model.layers.11.mlp.experts.8.gate_proj", "model.layers.11.mlp.experts.9.gate_proj", "model.layers.11.mlp.experts.10.gate_proj", "model.layers.11.mlp.experts.11.gate_proj", "model.layers.11.mlp.experts.12.gate_proj", "model.layers.11.mlp.experts.13.gate_proj", "model.layers.11.mlp.experts.14.gate_proj", "model.layers.11.mlp.experts.15.gate_proj", "model.layers.11.mlp.experts.16.gate_proj", "model.layers.11.mlp.experts.17.gate_proj", "model.layers.11.mlp.experts.18.gate_proj", "model.layers.11.mlp.experts.19.gate_proj", "model.layers.11.mlp.experts.20.gate_proj", "model.layers.11.mlp.experts.21.gate_proj", "model.layers.11.mlp.experts.22.gate_proj", "model.layers.11.mlp.experts.23.gate_proj", "model.layers.11.mlp.experts.24.gate_proj", "model.layers.11.mlp.experts.25.gate_proj", "model.layers.11.mlp.experts.26.gate_proj", "model.layers.11.mlp.experts.27.gate_proj", "model.layers.11.mlp.experts.28.gate_proj", "model.layers.11.mlp.experts.29.gate_proj", "model.layers.11.mlp.experts.30.gate_proj", "model.layers.11.mlp.experts.31.gate_proj", "model.layers.11.mlp.experts.32.gate_proj", "model.layers.11.mlp.experts.33.gate_proj", "model.layers.11.mlp.experts.34.gate_proj", "model.layers.11.mlp.experts.35.gate_proj", "model.layers.11.mlp.experts.36.gate_proj", "model.layers.11.mlp.experts.37.gate_proj", "model.layers.11.mlp.experts.38.gate_proj", "model.layers.11.mlp.experts.39.gate_proj", "model.layers.11.mlp.experts.40.gate_proj", "model.layers.11.mlp.experts.41.gate_proj", "model.layers.11.mlp.experts.42.gate_proj", "model.layers.11.mlp.experts.43.gate_proj", "model.layers.11.mlp.experts.44.gate_proj", "model.layers.11.mlp.experts.45.gate_proj", "model.layers.11.mlp.experts.46.gate_proj", "model.layers.11.mlp.experts.47.gate_proj", "model.layers.11.mlp.experts.48.gate_proj", "model.layers.11.mlp.experts.49.gate_proj", "model.layers.11.mlp.experts.50.gate_proj", "model.layers.11.mlp.experts.51.gate_proj", "model.layers.11.mlp.experts.52.gate_proj", "model.layers.11.mlp.experts.53.gate_proj", "model.layers.11.mlp.experts.54.gate_proj", "model.layers.11.mlp.experts.55.gate_proj", "model.layers.11.mlp.experts.56.gate_proj", "model.layers.11.mlp.experts.57.gate_proj", "model.layers.11.mlp.experts.58.gate_proj", "model.layers.11.mlp.experts.59.gate_proj", "model.layers.11.mlp.experts.60.gate_proj", "model.layers.11.mlp.experts.61.gate_proj", "model.layers.11.mlp.experts.62.gate_proj", "model.layers.11.mlp.experts.63.gate_proj", "model.layers.11.mlp.experts.64.gate_proj", "model.layers.11.mlp.experts.65.gate_proj", "model.layers.11.mlp.experts.66.gate_proj", "model.layers.11.mlp.experts.67.gate_proj", "model.layers.11.mlp.experts.68.gate_proj", "model.layers.11.mlp.experts.69.gate_proj", "model.layers.11.mlp.experts.70.gate_proj", "model.layers.11.mlp.experts.71.gate_proj", "model.layers.11.mlp.experts.72.gate_proj", "model.layers.11.mlp.experts.73.gate_proj", "model.layers.11.mlp.experts.74.gate_proj", "model.layers.11.mlp.experts.75.gate_proj", "model.layers.11.mlp.experts.76.gate_proj", "model.layers.11.mlp.experts.77.gate_proj", "model.layers.11.mlp.experts.78.gate_proj", "model.layers.11.mlp.experts.79.gate_proj", "model.layers.11.mlp.experts.80.gate_proj", "model.layers.11.mlp.experts.81.gate_proj", "model.layers.11.mlp.experts.82.gate_proj", "model.layers.11.mlp.experts.83.gate_proj", "model.layers.11.mlp.experts.84.gate_proj", "model.layers.11.mlp.experts.85.gate_proj", "model.layers.11.mlp.experts.86.gate_proj", "model.layers.11.mlp.experts.87.gate_proj", "model.layers.11.mlp.experts.88.gate_proj", "model.layers.11.mlp.experts.89.gate_proj", "model.layers.11.mlp.experts.90.gate_proj", "model.layers.11.mlp.experts.91.gate_proj", "model.layers.11.mlp.experts.92.gate_proj", "model.layers.11.mlp.experts.93.gate_proj", "model.layers.11.mlp.experts.94.gate_proj", "model.layers.11.mlp.experts.95.gate_proj", "model.layers.11.mlp.experts.96.gate_proj", "model.layers.11.mlp.experts.97.gate_proj", "model.layers.11.mlp.experts.98.gate_proj", "model.layers.11.mlp.experts.99.gate_proj", "model.layers.11.mlp.experts.100.gate_proj", "model.layers.11.mlp.experts.101.gate_proj", "model.layers.11.mlp.experts.102.gate_proj", "model.layers.11.mlp.experts.103.gate_proj", "model.layers.11.mlp.experts.104.gate_proj", "model.layers.11.mlp.experts.105.gate_proj", "model.layers.11.mlp.experts.106.gate_proj", "model.layers.11.mlp.experts.107.gate_proj", "model.layers.11.mlp.experts.108.gate_proj", "model.layers.11.mlp.experts.109.gate_proj", "model.layers.11.mlp.experts.110.gate_proj", "model.layers.11.mlp.experts.111.gate_proj", "model.layers.11.mlp.experts.112.gate_proj", "model.layers.11.mlp.experts.113.gate_proj", "model.layers.11.mlp.experts.114.gate_proj", "model.layers.11.mlp.experts.115.gate_proj", "model.layers.11.mlp.experts.116.gate_proj", "model.layers.11.mlp.experts.117.gate_proj", "model.layers.11.mlp.experts.118.gate_proj", "model.layers.11.mlp.experts.119.gate_proj", "model.layers.11.mlp.experts.120.gate_proj", "model.layers.11.mlp.experts.121.gate_proj", "model.layers.11.mlp.experts.122.gate_proj", "model.layers.11.mlp.experts.123.gate_proj", "model.layers.11.mlp.experts.124.gate_proj", "model.layers.11.mlp.experts.125.gate_proj", "model.layers.11.mlp.experts.126.gate_proj", "model.layers.11.mlp.experts.127.gate_proj", "model.layers.11.mlp.experts.128.gate_proj", "model.layers.11.mlp.experts.129.gate_proj", "model.layers.11.mlp.experts.130.gate_proj", "model.layers.11.mlp.experts.131.gate_proj", "model.layers.11.mlp.experts.132.gate_proj", "model.layers.11.mlp.experts.133.gate_proj", "model.layers.11.mlp.experts.134.gate_proj", "model.layers.11.mlp.experts.135.gate_proj", "model.layers.11.mlp.experts.136.gate_proj", "model.layers.11.mlp.experts.137.gate_proj", "model.layers.11.mlp.experts.138.gate_proj", "model.layers.11.mlp.experts.139.gate_proj", "model.layers.11.mlp.experts.140.gate_proj", "model.layers.11.mlp.experts.141.gate_proj", "model.layers.11.mlp.experts.142.gate_proj", "model.layers.11.mlp.experts.143.gate_proj", "model.layers.11.mlp.experts.144.gate_proj", "model.layers.11.mlp.experts.145.gate_proj", "model.layers.11.mlp.experts.146.gate_proj", "model.layers.11.mlp.experts.147.gate_proj", "model.layers.11.mlp.experts.148.gate_proj", "model.layers.11.mlp.experts.149.gate_proj", "model.layers.11.mlp.experts.150.gate_proj", "model.layers.11.mlp.experts.151.gate_proj", "model.layers.11.mlp.experts.152.gate_proj", "model.layers.11.mlp.experts.153.gate_proj", "model.layers.11.mlp.experts.154.gate_proj", "model.layers.11.mlp.experts.155.gate_proj", "model.layers.11.mlp.experts.156.gate_proj", "model.layers.11.mlp.experts.157.gate_proj", "model.layers.11.mlp.experts.158.gate_proj", "model.layers.11.mlp.experts.159.gate_proj", "model.layers.11.mlp.experts.0.up_proj", "model.layers.11.mlp.experts.1.up_proj", "model.layers.11.mlp.experts.2.up_proj", "model.layers.11.mlp.experts.3.up_proj", "model.layers.11.mlp.experts.4.up_proj", "model.layers.11.mlp.experts.5.up_proj", "model.layers.11.mlp.experts.6.up_proj", "model.layers.11.mlp.experts.7.up_proj", "model.layers.11.mlp.experts.8.up_proj", "model.layers.11.mlp.experts.9.up_proj", "model.layers.11.mlp.experts.10.up_proj", "model.layers.11.mlp.experts.11.up_proj", "model.layers.11.mlp.experts.12.up_proj", "model.layers.11.mlp.experts.13.up_proj", "model.layers.11.mlp.experts.14.up_proj", "model.layers.11.mlp.experts.15.up_proj", "model.layers.11.mlp.experts.16.up_proj", "model.layers.11.mlp.experts.17.up_proj", "model.layers.11.mlp.experts.18.up_proj", "model.layers.11.mlp.experts.19.up_proj", "model.layers.11.mlp.experts.20.up_proj", "model.layers.11.mlp.experts.21.up_proj", "model.layers.11.mlp.experts.22.up_proj", "model.layers.11.mlp.experts.23.up_proj", "model.layers.11.mlp.experts.24.up_proj", "model.layers.11.mlp.experts.25.up_proj", "model.layers.11.mlp.experts.26.up_proj", "model.layers.11.mlp.experts.27.up_proj", "model.layers.11.mlp.experts.28.up_proj", "model.layers.11.mlp.experts.29.up_proj", "model.layers.11.mlp.experts.30.up_proj", "model.layers.11.mlp.experts.31.up_proj", "model.layers.11.mlp.experts.32.up_proj", "model.layers.11.mlp.experts.33.up_proj", "model.layers.11.mlp.experts.34.up_proj", "model.layers.11.mlp.experts.35.up_proj", "model.layers.11.mlp.experts.36.up_proj", "model.layers.11.mlp.experts.37.up_proj", "model.layers.11.mlp.experts.38.up_proj", "model.layers.11.mlp.experts.39.up_proj", "model.layers.11.mlp.experts.40.up_proj", "model.layers.11.mlp.experts.41.up_proj", "model.layers.11.mlp.experts.42.up_proj", "model.layers.11.mlp.experts.43.up_proj", "model.layers.11.mlp.experts.44.up_proj", "model.layers.11.mlp.experts.45.up_proj", "model.layers.11.mlp.experts.46.up_proj", "model.layers.11.mlp.experts.47.up_proj", "model.layers.11.mlp.experts.48.up_proj", "model.layers.11.mlp.experts.49.up_proj", "model.layers.11.mlp.experts.50.up_proj", "model.layers.11.mlp.experts.51.up_proj", "model.layers.11.mlp.experts.52.up_proj", "model.layers.11.mlp.experts.53.up_proj", "model.layers.11.mlp.experts.54.up_proj", "model.layers.11.mlp.experts.55.up_proj", "model.layers.11.mlp.experts.56.up_proj", "model.layers.11.mlp.experts.57.up_proj", "model.layers.11.mlp.experts.58.up_proj", "model.layers.11.mlp.experts.59.up_proj", "model.layers.11.mlp.experts.60.up_proj", "model.layers.11.mlp.experts.61.up_proj", "model.layers.11.mlp.experts.62.up_proj", "model.layers.11.mlp.experts.63.up_proj", "model.layers.11.mlp.experts.64.up_proj", "model.layers.11.mlp.experts.65.up_proj", "model.layers.11.mlp.experts.66.up_proj", "model.layers.11.mlp.experts.67.up_proj", "model.layers.11.mlp.experts.68.up_proj", "model.layers.11.mlp.experts.69.up_proj", "model.layers.11.mlp.experts.70.up_proj", "model.layers.11.mlp.experts.71.up_proj", "model.layers.11.mlp.experts.72.up_proj", "model.layers.11.mlp.experts.73.up_proj", "model.layers.11.mlp.experts.74.up_proj", "model.layers.11.mlp.experts.75.up_proj", "model.layers.11.mlp.experts.76.up_proj", "model.layers.11.mlp.experts.77.up_proj", "model.layers.11.mlp.experts.78.up_proj", "model.layers.11.mlp.experts.79.up_proj", "model.layers.11.mlp.experts.80.up_proj", "model.layers.11.mlp.experts.81.up_proj", "model.layers.11.mlp.experts.82.up_proj", "model.layers.11.mlp.experts.83.up_proj", "model.layers.11.mlp.experts.84.up_proj", "model.layers.11.mlp.experts.85.up_proj", "model.layers.11.mlp.experts.86.up_proj", "model.layers.11.mlp.experts.87.up_proj", "model.layers.11.mlp.experts.88.up_proj", "model.layers.11.mlp.experts.89.up_proj", "model.layers.11.mlp.experts.90.up_proj", "model.layers.11.mlp.experts.91.up_proj", "model.layers.11.mlp.experts.92.up_proj", "model.layers.11.mlp.experts.93.up_proj", "model.layers.11.mlp.experts.94.up_proj", "model.layers.11.mlp.experts.95.up_proj", "model.layers.11.mlp.experts.96.up_proj", "model.layers.11.mlp.experts.97.up_proj", "model.layers.11.mlp.experts.98.up_proj", "model.layers.11.mlp.experts.99.up_proj", "model.layers.11.mlp.experts.100.up_proj", "model.layers.11.mlp.experts.101.up_proj", "model.layers.11.mlp.experts.102.up_proj", "model.layers.11.mlp.experts.103.up_proj", "model.layers.11.mlp.experts.104.up_proj", "model.layers.11.mlp.experts.105.up_proj", "model.layers.11.mlp.experts.106.up_proj", "model.layers.11.mlp.experts.107.up_proj", "model.layers.11.mlp.experts.108.up_proj", "model.layers.11.mlp.experts.109.up_proj", "model.layers.11.mlp.experts.110.up_proj", "model.layers.11.mlp.experts.111.up_proj", "model.layers.11.mlp.experts.112.up_proj", "model.layers.11.mlp.experts.113.up_proj", "model.layers.11.mlp.experts.114.up_proj", "model.layers.11.mlp.experts.115.up_proj", "model.layers.11.mlp.experts.116.up_proj", "model.layers.11.mlp.experts.117.up_proj", "model.layers.11.mlp.experts.118.up_proj", "model.layers.11.mlp.experts.119.up_proj", "model.layers.11.mlp.experts.120.up_proj", "model.layers.11.mlp.experts.121.up_proj", "model.layers.11.mlp.experts.122.up_proj", "model.layers.11.mlp.experts.123.up_proj", "model.layers.11.mlp.experts.124.up_proj", "model.layers.11.mlp.experts.125.up_proj", "model.layers.11.mlp.experts.126.up_proj", "model.layers.11.mlp.experts.127.up_proj", "model.layers.11.mlp.experts.128.up_proj", "model.layers.11.mlp.experts.129.up_proj", "model.layers.11.mlp.experts.130.up_proj", "model.layers.11.mlp.experts.131.up_proj", "model.layers.11.mlp.experts.132.up_proj", "model.layers.11.mlp.experts.133.up_proj", "model.layers.11.mlp.experts.134.up_proj", "model.layers.11.mlp.experts.135.up_proj", "model.layers.11.mlp.experts.136.up_proj", "model.layers.11.mlp.experts.137.up_proj", "model.layers.11.mlp.experts.138.up_proj", "model.layers.11.mlp.experts.139.up_proj", "model.layers.11.mlp.experts.140.up_proj", "model.layers.11.mlp.experts.141.up_proj", "model.layers.11.mlp.experts.142.up_proj", "model.layers.11.mlp.experts.143.up_proj", "model.layers.11.mlp.experts.144.up_proj", "model.layers.11.mlp.experts.145.up_proj", "model.layers.11.mlp.experts.146.up_proj", "model.layers.11.mlp.experts.147.up_proj", "model.layers.11.mlp.experts.148.up_proj", "model.layers.11.mlp.experts.149.up_proj", "model.layers.11.mlp.experts.150.up_proj", "model.layers.11.mlp.experts.151.up_proj", "model.layers.11.mlp.experts.152.up_proj", "model.layers.11.mlp.experts.153.up_proj", "model.layers.11.mlp.experts.154.up_proj", "model.layers.11.mlp.experts.155.up_proj", "model.layers.11.mlp.experts.156.up_proj", "model.layers.11.mlp.experts.157.up_proj", "model.layers.11.mlp.experts.158.up_proj", "model.layers.11.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.595575399231168e-05, "dbits": 5033164800 } ] }, { "idx": 68, "layers": [ "model.layers.11.mlp.experts.0.down_proj", "model.layers.11.mlp.experts.1.down_proj", "model.layers.11.mlp.experts.2.down_proj", "model.layers.11.mlp.experts.3.down_proj", "model.layers.11.mlp.experts.4.down_proj", "model.layers.11.mlp.experts.5.down_proj", "model.layers.11.mlp.experts.6.down_proj", "model.layers.11.mlp.experts.7.down_proj", "model.layers.11.mlp.experts.8.down_proj", "model.layers.11.mlp.experts.9.down_proj", "model.layers.11.mlp.experts.10.down_proj", "model.layers.11.mlp.experts.11.down_proj", "model.layers.11.mlp.experts.12.down_proj", "model.layers.11.mlp.experts.13.down_proj", "model.layers.11.mlp.experts.14.down_proj", "model.layers.11.mlp.experts.15.down_proj", "model.layers.11.mlp.experts.16.down_proj", "model.layers.11.mlp.experts.17.down_proj", "model.layers.11.mlp.experts.18.down_proj", "model.layers.11.mlp.experts.19.down_proj", "model.layers.11.mlp.experts.20.down_proj", "model.layers.11.mlp.experts.21.down_proj", "model.layers.11.mlp.experts.22.down_proj", "model.layers.11.mlp.experts.23.down_proj", "model.layers.11.mlp.experts.24.down_proj", "model.layers.11.mlp.experts.25.down_proj", "model.layers.11.mlp.experts.26.down_proj", "model.layers.11.mlp.experts.27.down_proj", "model.layers.11.mlp.experts.28.down_proj", "model.layers.11.mlp.experts.29.down_proj", "model.layers.11.mlp.experts.30.down_proj", "model.layers.11.mlp.experts.31.down_proj", "model.layers.11.mlp.experts.32.down_proj", "model.layers.11.mlp.experts.33.down_proj", "model.layers.11.mlp.experts.34.down_proj", "model.layers.11.mlp.experts.35.down_proj", "model.layers.11.mlp.experts.36.down_proj", "model.layers.11.mlp.experts.37.down_proj", "model.layers.11.mlp.experts.38.down_proj", "model.layers.11.mlp.experts.39.down_proj", "model.layers.11.mlp.experts.40.down_proj", "model.layers.11.mlp.experts.41.down_proj", "model.layers.11.mlp.experts.42.down_proj", "model.layers.11.mlp.experts.43.down_proj", "model.layers.11.mlp.experts.44.down_proj", "model.layers.11.mlp.experts.45.down_proj", "model.layers.11.mlp.experts.46.down_proj", "model.layers.11.mlp.experts.47.down_proj", "model.layers.11.mlp.experts.48.down_proj", "model.layers.11.mlp.experts.49.down_proj", "model.layers.11.mlp.experts.50.down_proj", "model.layers.11.mlp.experts.51.down_proj", "model.layers.11.mlp.experts.52.down_proj", "model.layers.11.mlp.experts.53.down_proj", "model.layers.11.mlp.experts.54.down_proj", "model.layers.11.mlp.experts.55.down_proj", "model.layers.11.mlp.experts.56.down_proj", "model.layers.11.mlp.experts.57.down_proj", "model.layers.11.mlp.experts.58.down_proj", "model.layers.11.mlp.experts.59.down_proj", "model.layers.11.mlp.experts.60.down_proj", "model.layers.11.mlp.experts.61.down_proj", "model.layers.11.mlp.experts.62.down_proj", "model.layers.11.mlp.experts.63.down_proj", "model.layers.11.mlp.experts.64.down_proj", "model.layers.11.mlp.experts.65.down_proj", "model.layers.11.mlp.experts.66.down_proj", "model.layers.11.mlp.experts.67.down_proj", "model.layers.11.mlp.experts.68.down_proj", "model.layers.11.mlp.experts.69.down_proj", "model.layers.11.mlp.experts.70.down_proj", "model.layers.11.mlp.experts.71.down_proj", "model.layers.11.mlp.experts.72.down_proj", "model.layers.11.mlp.experts.73.down_proj", "model.layers.11.mlp.experts.74.down_proj", "model.layers.11.mlp.experts.75.down_proj", "model.layers.11.mlp.experts.76.down_proj", "model.layers.11.mlp.experts.77.down_proj", "model.layers.11.mlp.experts.78.down_proj", "model.layers.11.mlp.experts.79.down_proj", "model.layers.11.mlp.experts.80.down_proj", "model.layers.11.mlp.experts.81.down_proj", "model.layers.11.mlp.experts.82.down_proj", "model.layers.11.mlp.experts.83.down_proj", "model.layers.11.mlp.experts.84.down_proj", "model.layers.11.mlp.experts.85.down_proj", "model.layers.11.mlp.experts.86.down_proj", "model.layers.11.mlp.experts.87.down_proj", "model.layers.11.mlp.experts.88.down_proj", "model.layers.11.mlp.experts.89.down_proj", "model.layers.11.mlp.experts.90.down_proj", "model.layers.11.mlp.experts.91.down_proj", "model.layers.11.mlp.experts.92.down_proj", "model.layers.11.mlp.experts.93.down_proj", "model.layers.11.mlp.experts.94.down_proj", "model.layers.11.mlp.experts.95.down_proj", "model.layers.11.mlp.experts.96.down_proj", "model.layers.11.mlp.experts.97.down_proj", "model.layers.11.mlp.experts.98.down_proj", "model.layers.11.mlp.experts.99.down_proj", "model.layers.11.mlp.experts.100.down_proj", "model.layers.11.mlp.experts.101.down_proj", "model.layers.11.mlp.experts.102.down_proj", "model.layers.11.mlp.experts.103.down_proj", "model.layers.11.mlp.experts.104.down_proj", "model.layers.11.mlp.experts.105.down_proj", "model.layers.11.mlp.experts.106.down_proj", "model.layers.11.mlp.experts.107.down_proj", "model.layers.11.mlp.experts.108.down_proj", "model.layers.11.mlp.experts.109.down_proj", "model.layers.11.mlp.experts.110.down_proj", "model.layers.11.mlp.experts.111.down_proj", "model.layers.11.mlp.experts.112.down_proj", "model.layers.11.mlp.experts.113.down_proj", "model.layers.11.mlp.experts.114.down_proj", "model.layers.11.mlp.experts.115.down_proj", "model.layers.11.mlp.experts.116.down_proj", "model.layers.11.mlp.experts.117.down_proj", "model.layers.11.mlp.experts.118.down_proj", "model.layers.11.mlp.experts.119.down_proj", "model.layers.11.mlp.experts.120.down_proj", "model.layers.11.mlp.experts.121.down_proj", "model.layers.11.mlp.experts.122.down_proj", "model.layers.11.mlp.experts.123.down_proj", "model.layers.11.mlp.experts.124.down_proj", "model.layers.11.mlp.experts.125.down_proj", "model.layers.11.mlp.experts.126.down_proj", "model.layers.11.mlp.experts.127.down_proj", "model.layers.11.mlp.experts.128.down_proj", "model.layers.11.mlp.experts.129.down_proj", "model.layers.11.mlp.experts.130.down_proj", "model.layers.11.mlp.experts.131.down_proj", "model.layers.11.mlp.experts.132.down_proj", "model.layers.11.mlp.experts.133.down_proj", "model.layers.11.mlp.experts.134.down_proj", "model.layers.11.mlp.experts.135.down_proj", "model.layers.11.mlp.experts.136.down_proj", "model.layers.11.mlp.experts.137.down_proj", "model.layers.11.mlp.experts.138.down_proj", "model.layers.11.mlp.experts.139.down_proj", "model.layers.11.mlp.experts.140.down_proj", "model.layers.11.mlp.experts.141.down_proj", "model.layers.11.mlp.experts.142.down_proj", "model.layers.11.mlp.experts.143.down_proj", "model.layers.11.mlp.experts.144.down_proj", "model.layers.11.mlp.experts.145.down_proj", "model.layers.11.mlp.experts.146.down_proj", "model.layers.11.mlp.experts.147.down_proj", "model.layers.11.mlp.experts.148.down_proj", "model.layers.11.mlp.experts.149.down_proj", "model.layers.11.mlp.experts.150.down_proj", "model.layers.11.mlp.experts.151.down_proj", "model.layers.11.mlp.experts.152.down_proj", "model.layers.11.mlp.experts.153.down_proj", "model.layers.11.mlp.experts.154.down_proj", "model.layers.11.mlp.experts.155.down_proj", "model.layers.11.mlp.experts.156.down_proj", "model.layers.11.mlp.experts.157.down_proj", "model.layers.11.mlp.experts.158.down_proj", "model.layers.11.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -2.0764238433912418e-05, "dbits": 2516582400 } ] }, { "idx": 69, "layers": [ "model.layers.12.self_attn.q_proj" ], "candidates": [ { "dkld": 1.556334900669758e-05, "dbits": 125829120 } ] }, { "idx": 70, "layers": [ "model.layers.12.self_attn.k_proj", "model.layers.12.self_attn.v_proj" ], "candidates": [ { "dkld": -1.927549019455912e-05, "dbits": 20971520 } ] }, { "idx": 71, "layers": [ "model.layers.12.self_attn.o_proj" ], "candidates": [ { "dkld": -1.5960531891323604e-05, "dbits": 125829120 } ] }, { "idx": 72, "layers": [ "model.layers.12.mlp.shared_experts.gate_proj", "model.layers.12.mlp.shared_experts.up_proj", "model.layers.12.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -7.500406354665539e-07, "dbits": 47185920 } ] }, { "idx": 73, "layers": [ "model.layers.12.mlp.experts.0.gate_proj", "model.layers.12.mlp.experts.1.gate_proj", "model.layers.12.mlp.experts.2.gate_proj", "model.layers.12.mlp.experts.3.gate_proj", "model.layers.12.mlp.experts.4.gate_proj", "model.layers.12.mlp.experts.5.gate_proj", "model.layers.12.mlp.experts.6.gate_proj", "model.layers.12.mlp.experts.7.gate_proj", "model.layers.12.mlp.experts.8.gate_proj", "model.layers.12.mlp.experts.9.gate_proj", "model.layers.12.mlp.experts.10.gate_proj", "model.layers.12.mlp.experts.11.gate_proj", "model.layers.12.mlp.experts.12.gate_proj", "model.layers.12.mlp.experts.13.gate_proj", "model.layers.12.mlp.experts.14.gate_proj", "model.layers.12.mlp.experts.15.gate_proj", "model.layers.12.mlp.experts.16.gate_proj", "model.layers.12.mlp.experts.17.gate_proj", "model.layers.12.mlp.experts.18.gate_proj", "model.layers.12.mlp.experts.19.gate_proj", "model.layers.12.mlp.experts.20.gate_proj", "model.layers.12.mlp.experts.21.gate_proj", "model.layers.12.mlp.experts.22.gate_proj", "model.layers.12.mlp.experts.23.gate_proj", "model.layers.12.mlp.experts.24.gate_proj", "model.layers.12.mlp.experts.25.gate_proj", "model.layers.12.mlp.experts.26.gate_proj", "model.layers.12.mlp.experts.27.gate_proj", "model.layers.12.mlp.experts.28.gate_proj", "model.layers.12.mlp.experts.29.gate_proj", "model.layers.12.mlp.experts.30.gate_proj", "model.layers.12.mlp.experts.31.gate_proj", "model.layers.12.mlp.experts.32.gate_proj", "model.layers.12.mlp.experts.33.gate_proj", "model.layers.12.mlp.experts.34.gate_proj", "model.layers.12.mlp.experts.35.gate_proj", "model.layers.12.mlp.experts.36.gate_proj", "model.layers.12.mlp.experts.37.gate_proj", "model.layers.12.mlp.experts.38.gate_proj", "model.layers.12.mlp.experts.39.gate_proj", "model.layers.12.mlp.experts.40.gate_proj", "model.layers.12.mlp.experts.41.gate_proj", "model.layers.12.mlp.experts.42.gate_proj", "model.layers.12.mlp.experts.43.gate_proj", "model.layers.12.mlp.experts.44.gate_proj", "model.layers.12.mlp.experts.45.gate_proj", "model.layers.12.mlp.experts.46.gate_proj", "model.layers.12.mlp.experts.47.gate_proj", "model.layers.12.mlp.experts.48.gate_proj", "model.layers.12.mlp.experts.49.gate_proj", "model.layers.12.mlp.experts.50.gate_proj", "model.layers.12.mlp.experts.51.gate_proj", "model.layers.12.mlp.experts.52.gate_proj", "model.layers.12.mlp.experts.53.gate_proj", "model.layers.12.mlp.experts.54.gate_proj", "model.layers.12.mlp.experts.55.gate_proj", "model.layers.12.mlp.experts.56.gate_proj", "model.layers.12.mlp.experts.57.gate_proj", "model.layers.12.mlp.experts.58.gate_proj", "model.layers.12.mlp.experts.59.gate_proj", "model.layers.12.mlp.experts.60.gate_proj", "model.layers.12.mlp.experts.61.gate_proj", "model.layers.12.mlp.experts.62.gate_proj", "model.layers.12.mlp.experts.63.gate_proj", "model.layers.12.mlp.experts.64.gate_proj", "model.layers.12.mlp.experts.65.gate_proj", "model.layers.12.mlp.experts.66.gate_proj", "model.layers.12.mlp.experts.67.gate_proj", "model.layers.12.mlp.experts.68.gate_proj", "model.layers.12.mlp.experts.69.gate_proj", "model.layers.12.mlp.experts.70.gate_proj", "model.layers.12.mlp.experts.71.gate_proj", "model.layers.12.mlp.experts.72.gate_proj", "model.layers.12.mlp.experts.73.gate_proj", "model.layers.12.mlp.experts.74.gate_proj", "model.layers.12.mlp.experts.75.gate_proj", "model.layers.12.mlp.experts.76.gate_proj", "model.layers.12.mlp.experts.77.gate_proj", "model.layers.12.mlp.experts.78.gate_proj", "model.layers.12.mlp.experts.79.gate_proj", "model.layers.12.mlp.experts.80.gate_proj", "model.layers.12.mlp.experts.81.gate_proj", "model.layers.12.mlp.experts.82.gate_proj", "model.layers.12.mlp.experts.83.gate_proj", "model.layers.12.mlp.experts.84.gate_proj", "model.layers.12.mlp.experts.85.gate_proj", "model.layers.12.mlp.experts.86.gate_proj", "model.layers.12.mlp.experts.87.gate_proj", "model.layers.12.mlp.experts.88.gate_proj", "model.layers.12.mlp.experts.89.gate_proj", "model.layers.12.mlp.experts.90.gate_proj", "model.layers.12.mlp.experts.91.gate_proj", "model.layers.12.mlp.experts.92.gate_proj", "model.layers.12.mlp.experts.93.gate_proj", "model.layers.12.mlp.experts.94.gate_proj", "model.layers.12.mlp.experts.95.gate_proj", "model.layers.12.mlp.experts.96.gate_proj", "model.layers.12.mlp.experts.97.gate_proj", "model.layers.12.mlp.experts.98.gate_proj", "model.layers.12.mlp.experts.99.gate_proj", "model.layers.12.mlp.experts.100.gate_proj", "model.layers.12.mlp.experts.101.gate_proj", "model.layers.12.mlp.experts.102.gate_proj", "model.layers.12.mlp.experts.103.gate_proj", "model.layers.12.mlp.experts.104.gate_proj", "model.layers.12.mlp.experts.105.gate_proj", "model.layers.12.mlp.experts.106.gate_proj", "model.layers.12.mlp.experts.107.gate_proj", "model.layers.12.mlp.experts.108.gate_proj", "model.layers.12.mlp.experts.109.gate_proj", "model.layers.12.mlp.experts.110.gate_proj", "model.layers.12.mlp.experts.111.gate_proj", "model.layers.12.mlp.experts.112.gate_proj", "model.layers.12.mlp.experts.113.gate_proj", "model.layers.12.mlp.experts.114.gate_proj", "model.layers.12.mlp.experts.115.gate_proj", "model.layers.12.mlp.experts.116.gate_proj", "model.layers.12.mlp.experts.117.gate_proj", "model.layers.12.mlp.experts.118.gate_proj", "model.layers.12.mlp.experts.119.gate_proj", "model.layers.12.mlp.experts.120.gate_proj", "model.layers.12.mlp.experts.121.gate_proj", "model.layers.12.mlp.experts.122.gate_proj", "model.layers.12.mlp.experts.123.gate_proj", "model.layers.12.mlp.experts.124.gate_proj", "model.layers.12.mlp.experts.125.gate_proj", "model.layers.12.mlp.experts.126.gate_proj", "model.layers.12.mlp.experts.127.gate_proj", "model.layers.12.mlp.experts.128.gate_proj", "model.layers.12.mlp.experts.129.gate_proj", "model.layers.12.mlp.experts.130.gate_proj", "model.layers.12.mlp.experts.131.gate_proj", "model.layers.12.mlp.experts.132.gate_proj", "model.layers.12.mlp.experts.133.gate_proj", "model.layers.12.mlp.experts.134.gate_proj", "model.layers.12.mlp.experts.135.gate_proj", "model.layers.12.mlp.experts.136.gate_proj", "model.layers.12.mlp.experts.137.gate_proj", "model.layers.12.mlp.experts.138.gate_proj", "model.layers.12.mlp.experts.139.gate_proj", "model.layers.12.mlp.experts.140.gate_proj", "model.layers.12.mlp.experts.141.gate_proj", "model.layers.12.mlp.experts.142.gate_proj", "model.layers.12.mlp.experts.143.gate_proj", "model.layers.12.mlp.experts.144.gate_proj", "model.layers.12.mlp.experts.145.gate_proj", "model.layers.12.mlp.experts.146.gate_proj", "model.layers.12.mlp.experts.147.gate_proj", "model.layers.12.mlp.experts.148.gate_proj", "model.layers.12.mlp.experts.149.gate_proj", "model.layers.12.mlp.experts.150.gate_proj", "model.layers.12.mlp.experts.151.gate_proj", "model.layers.12.mlp.experts.152.gate_proj", "model.layers.12.mlp.experts.153.gate_proj", "model.layers.12.mlp.experts.154.gate_proj", "model.layers.12.mlp.experts.155.gate_proj", "model.layers.12.mlp.experts.156.gate_proj", "model.layers.12.mlp.experts.157.gate_proj", "model.layers.12.mlp.experts.158.gate_proj", "model.layers.12.mlp.experts.159.gate_proj", "model.layers.12.mlp.experts.0.up_proj", "model.layers.12.mlp.experts.1.up_proj", "model.layers.12.mlp.experts.2.up_proj", "model.layers.12.mlp.experts.3.up_proj", "model.layers.12.mlp.experts.4.up_proj", "model.layers.12.mlp.experts.5.up_proj", "model.layers.12.mlp.experts.6.up_proj", "model.layers.12.mlp.experts.7.up_proj", "model.layers.12.mlp.experts.8.up_proj", "model.layers.12.mlp.experts.9.up_proj", "model.layers.12.mlp.experts.10.up_proj", "model.layers.12.mlp.experts.11.up_proj", "model.layers.12.mlp.experts.12.up_proj", "model.layers.12.mlp.experts.13.up_proj", "model.layers.12.mlp.experts.14.up_proj", "model.layers.12.mlp.experts.15.up_proj", "model.layers.12.mlp.experts.16.up_proj", "model.layers.12.mlp.experts.17.up_proj", "model.layers.12.mlp.experts.18.up_proj", "model.layers.12.mlp.experts.19.up_proj", "model.layers.12.mlp.experts.20.up_proj", "model.layers.12.mlp.experts.21.up_proj", "model.layers.12.mlp.experts.22.up_proj", "model.layers.12.mlp.experts.23.up_proj", "model.layers.12.mlp.experts.24.up_proj", "model.layers.12.mlp.experts.25.up_proj", "model.layers.12.mlp.experts.26.up_proj", "model.layers.12.mlp.experts.27.up_proj", "model.layers.12.mlp.experts.28.up_proj", "model.layers.12.mlp.experts.29.up_proj", "model.layers.12.mlp.experts.30.up_proj", "model.layers.12.mlp.experts.31.up_proj", "model.layers.12.mlp.experts.32.up_proj", "model.layers.12.mlp.experts.33.up_proj", "model.layers.12.mlp.experts.34.up_proj", "model.layers.12.mlp.experts.35.up_proj", "model.layers.12.mlp.experts.36.up_proj", "model.layers.12.mlp.experts.37.up_proj", "model.layers.12.mlp.experts.38.up_proj", "model.layers.12.mlp.experts.39.up_proj", "model.layers.12.mlp.experts.40.up_proj", "model.layers.12.mlp.experts.41.up_proj", "model.layers.12.mlp.experts.42.up_proj", "model.layers.12.mlp.experts.43.up_proj", "model.layers.12.mlp.experts.44.up_proj", "model.layers.12.mlp.experts.45.up_proj", "model.layers.12.mlp.experts.46.up_proj", "model.layers.12.mlp.experts.47.up_proj", "model.layers.12.mlp.experts.48.up_proj", "model.layers.12.mlp.experts.49.up_proj", "model.layers.12.mlp.experts.50.up_proj", "model.layers.12.mlp.experts.51.up_proj", "model.layers.12.mlp.experts.52.up_proj", "model.layers.12.mlp.experts.53.up_proj", "model.layers.12.mlp.experts.54.up_proj", "model.layers.12.mlp.experts.55.up_proj", "model.layers.12.mlp.experts.56.up_proj", "model.layers.12.mlp.experts.57.up_proj", "model.layers.12.mlp.experts.58.up_proj", "model.layers.12.mlp.experts.59.up_proj", "model.layers.12.mlp.experts.60.up_proj", "model.layers.12.mlp.experts.61.up_proj", "model.layers.12.mlp.experts.62.up_proj", "model.layers.12.mlp.experts.63.up_proj", "model.layers.12.mlp.experts.64.up_proj", "model.layers.12.mlp.experts.65.up_proj", "model.layers.12.mlp.experts.66.up_proj", "model.layers.12.mlp.experts.67.up_proj", "model.layers.12.mlp.experts.68.up_proj", "model.layers.12.mlp.experts.69.up_proj", "model.layers.12.mlp.experts.70.up_proj", "model.layers.12.mlp.experts.71.up_proj", "model.layers.12.mlp.experts.72.up_proj", "model.layers.12.mlp.experts.73.up_proj", "model.layers.12.mlp.experts.74.up_proj", "model.layers.12.mlp.experts.75.up_proj", "model.layers.12.mlp.experts.76.up_proj", "model.layers.12.mlp.experts.77.up_proj", "model.layers.12.mlp.experts.78.up_proj", "model.layers.12.mlp.experts.79.up_proj", "model.layers.12.mlp.experts.80.up_proj", "model.layers.12.mlp.experts.81.up_proj", "model.layers.12.mlp.experts.82.up_proj", "model.layers.12.mlp.experts.83.up_proj", "model.layers.12.mlp.experts.84.up_proj", "model.layers.12.mlp.experts.85.up_proj", "model.layers.12.mlp.experts.86.up_proj", "model.layers.12.mlp.experts.87.up_proj", "model.layers.12.mlp.experts.88.up_proj", "model.layers.12.mlp.experts.89.up_proj", "model.layers.12.mlp.experts.90.up_proj", "model.layers.12.mlp.experts.91.up_proj", "model.layers.12.mlp.experts.92.up_proj", "model.layers.12.mlp.experts.93.up_proj", "model.layers.12.mlp.experts.94.up_proj", "model.layers.12.mlp.experts.95.up_proj", "model.layers.12.mlp.experts.96.up_proj", "model.layers.12.mlp.experts.97.up_proj", "model.layers.12.mlp.experts.98.up_proj", "model.layers.12.mlp.experts.99.up_proj", "model.layers.12.mlp.experts.100.up_proj", "model.layers.12.mlp.experts.101.up_proj", "model.layers.12.mlp.experts.102.up_proj", "model.layers.12.mlp.experts.103.up_proj", "model.layers.12.mlp.experts.104.up_proj", "model.layers.12.mlp.experts.105.up_proj", "model.layers.12.mlp.experts.106.up_proj", "model.layers.12.mlp.experts.107.up_proj", "model.layers.12.mlp.experts.108.up_proj", "model.layers.12.mlp.experts.109.up_proj", "model.layers.12.mlp.experts.110.up_proj", "model.layers.12.mlp.experts.111.up_proj", "model.layers.12.mlp.experts.112.up_proj", "model.layers.12.mlp.experts.113.up_proj", "model.layers.12.mlp.experts.114.up_proj", "model.layers.12.mlp.experts.115.up_proj", "model.layers.12.mlp.experts.116.up_proj", "model.layers.12.mlp.experts.117.up_proj", "model.layers.12.mlp.experts.118.up_proj", "model.layers.12.mlp.experts.119.up_proj", "model.layers.12.mlp.experts.120.up_proj", "model.layers.12.mlp.experts.121.up_proj", "model.layers.12.mlp.experts.122.up_proj", "model.layers.12.mlp.experts.123.up_proj", "model.layers.12.mlp.experts.124.up_proj", "model.layers.12.mlp.experts.125.up_proj", "model.layers.12.mlp.experts.126.up_proj", "model.layers.12.mlp.experts.127.up_proj", "model.layers.12.mlp.experts.128.up_proj", "model.layers.12.mlp.experts.129.up_proj", "model.layers.12.mlp.experts.130.up_proj", "model.layers.12.mlp.experts.131.up_proj", "model.layers.12.mlp.experts.132.up_proj", "model.layers.12.mlp.experts.133.up_proj", "model.layers.12.mlp.experts.134.up_proj", "model.layers.12.mlp.experts.135.up_proj", "model.layers.12.mlp.experts.136.up_proj", "model.layers.12.mlp.experts.137.up_proj", "model.layers.12.mlp.experts.138.up_proj", "model.layers.12.mlp.experts.139.up_proj", "model.layers.12.mlp.experts.140.up_proj", "model.layers.12.mlp.experts.141.up_proj", "model.layers.12.mlp.experts.142.up_proj", "model.layers.12.mlp.experts.143.up_proj", "model.layers.12.mlp.experts.144.up_proj", "model.layers.12.mlp.experts.145.up_proj", "model.layers.12.mlp.experts.146.up_proj", "model.layers.12.mlp.experts.147.up_proj", "model.layers.12.mlp.experts.148.up_proj", "model.layers.12.mlp.experts.149.up_proj", "model.layers.12.mlp.experts.150.up_proj", "model.layers.12.mlp.experts.151.up_proj", "model.layers.12.mlp.experts.152.up_proj", "model.layers.12.mlp.experts.153.up_proj", "model.layers.12.mlp.experts.154.up_proj", "model.layers.12.mlp.experts.155.up_proj", "model.layers.12.mlp.experts.156.up_proj", "model.layers.12.mlp.experts.157.up_proj", "model.layers.12.mlp.experts.158.up_proj", "model.layers.12.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.8794552306644575e-05, "dbits": 5033164800 } ] }, { "idx": 74, "layers": [ "model.layers.12.mlp.experts.0.down_proj", "model.layers.12.mlp.experts.1.down_proj", "model.layers.12.mlp.experts.2.down_proj", "model.layers.12.mlp.experts.3.down_proj", "model.layers.12.mlp.experts.4.down_proj", "model.layers.12.mlp.experts.5.down_proj", "model.layers.12.mlp.experts.6.down_proj", "model.layers.12.mlp.experts.7.down_proj", "model.layers.12.mlp.experts.8.down_proj", "model.layers.12.mlp.experts.9.down_proj", "model.layers.12.mlp.experts.10.down_proj", "model.layers.12.mlp.experts.11.down_proj", "model.layers.12.mlp.experts.12.down_proj", "model.layers.12.mlp.experts.13.down_proj", "model.layers.12.mlp.experts.14.down_proj", "model.layers.12.mlp.experts.15.down_proj", "model.layers.12.mlp.experts.16.down_proj", "model.layers.12.mlp.experts.17.down_proj", "model.layers.12.mlp.experts.18.down_proj", "model.layers.12.mlp.experts.19.down_proj", "model.layers.12.mlp.experts.20.down_proj", "model.layers.12.mlp.experts.21.down_proj", "model.layers.12.mlp.experts.22.down_proj", "model.layers.12.mlp.experts.23.down_proj", "model.layers.12.mlp.experts.24.down_proj", "model.layers.12.mlp.experts.25.down_proj", "model.layers.12.mlp.experts.26.down_proj", "model.layers.12.mlp.experts.27.down_proj", "model.layers.12.mlp.experts.28.down_proj", "model.layers.12.mlp.experts.29.down_proj", "model.layers.12.mlp.experts.30.down_proj", "model.layers.12.mlp.experts.31.down_proj", "model.layers.12.mlp.experts.32.down_proj", "model.layers.12.mlp.experts.33.down_proj", "model.layers.12.mlp.experts.34.down_proj", "model.layers.12.mlp.experts.35.down_proj", "model.layers.12.mlp.experts.36.down_proj", "model.layers.12.mlp.experts.37.down_proj", "model.layers.12.mlp.experts.38.down_proj", "model.layers.12.mlp.experts.39.down_proj", "model.layers.12.mlp.experts.40.down_proj", "model.layers.12.mlp.experts.41.down_proj", "model.layers.12.mlp.experts.42.down_proj", "model.layers.12.mlp.experts.43.down_proj", "model.layers.12.mlp.experts.44.down_proj", "model.layers.12.mlp.experts.45.down_proj", "model.layers.12.mlp.experts.46.down_proj", "model.layers.12.mlp.experts.47.down_proj", "model.layers.12.mlp.experts.48.down_proj", "model.layers.12.mlp.experts.49.down_proj", "model.layers.12.mlp.experts.50.down_proj", "model.layers.12.mlp.experts.51.down_proj", "model.layers.12.mlp.experts.52.down_proj", "model.layers.12.mlp.experts.53.down_proj", "model.layers.12.mlp.experts.54.down_proj", "model.layers.12.mlp.experts.55.down_proj", "model.layers.12.mlp.experts.56.down_proj", "model.layers.12.mlp.experts.57.down_proj", "model.layers.12.mlp.experts.58.down_proj", "model.layers.12.mlp.experts.59.down_proj", "model.layers.12.mlp.experts.60.down_proj", "model.layers.12.mlp.experts.61.down_proj", "model.layers.12.mlp.experts.62.down_proj", "model.layers.12.mlp.experts.63.down_proj", "model.layers.12.mlp.experts.64.down_proj", "model.layers.12.mlp.experts.65.down_proj", "model.layers.12.mlp.experts.66.down_proj", "model.layers.12.mlp.experts.67.down_proj", "model.layers.12.mlp.experts.68.down_proj", "model.layers.12.mlp.experts.69.down_proj", "model.layers.12.mlp.experts.70.down_proj", "model.layers.12.mlp.experts.71.down_proj", "model.layers.12.mlp.experts.72.down_proj", "model.layers.12.mlp.experts.73.down_proj", "model.layers.12.mlp.experts.74.down_proj", "model.layers.12.mlp.experts.75.down_proj", "model.layers.12.mlp.experts.76.down_proj", "model.layers.12.mlp.experts.77.down_proj", "model.layers.12.mlp.experts.78.down_proj", "model.layers.12.mlp.experts.79.down_proj", "model.layers.12.mlp.experts.80.down_proj", "model.layers.12.mlp.experts.81.down_proj", "model.layers.12.mlp.experts.82.down_proj", "model.layers.12.mlp.experts.83.down_proj", "model.layers.12.mlp.experts.84.down_proj", "model.layers.12.mlp.experts.85.down_proj", "model.layers.12.mlp.experts.86.down_proj", "model.layers.12.mlp.experts.87.down_proj", "model.layers.12.mlp.experts.88.down_proj", "model.layers.12.mlp.experts.89.down_proj", "model.layers.12.mlp.experts.90.down_proj", "model.layers.12.mlp.experts.91.down_proj", "model.layers.12.mlp.experts.92.down_proj", "model.layers.12.mlp.experts.93.down_proj", "model.layers.12.mlp.experts.94.down_proj", "model.layers.12.mlp.experts.95.down_proj", "model.layers.12.mlp.experts.96.down_proj", "model.layers.12.mlp.experts.97.down_proj", "model.layers.12.mlp.experts.98.down_proj", "model.layers.12.mlp.experts.99.down_proj", "model.layers.12.mlp.experts.100.down_proj", "model.layers.12.mlp.experts.101.down_proj", "model.layers.12.mlp.experts.102.down_proj", "model.layers.12.mlp.experts.103.down_proj", "model.layers.12.mlp.experts.104.down_proj", "model.layers.12.mlp.experts.105.down_proj", "model.layers.12.mlp.experts.106.down_proj", "model.layers.12.mlp.experts.107.down_proj", "model.layers.12.mlp.experts.108.down_proj", "model.layers.12.mlp.experts.109.down_proj", "model.layers.12.mlp.experts.110.down_proj", "model.layers.12.mlp.experts.111.down_proj", "model.layers.12.mlp.experts.112.down_proj", "model.layers.12.mlp.experts.113.down_proj", "model.layers.12.mlp.experts.114.down_proj", "model.layers.12.mlp.experts.115.down_proj", "model.layers.12.mlp.experts.116.down_proj", "model.layers.12.mlp.experts.117.down_proj", "model.layers.12.mlp.experts.118.down_proj", "model.layers.12.mlp.experts.119.down_proj", "model.layers.12.mlp.experts.120.down_proj", "model.layers.12.mlp.experts.121.down_proj", "model.layers.12.mlp.experts.122.down_proj", "model.layers.12.mlp.experts.123.down_proj", "model.layers.12.mlp.experts.124.down_proj", "model.layers.12.mlp.experts.125.down_proj", "model.layers.12.mlp.experts.126.down_proj", "model.layers.12.mlp.experts.127.down_proj", "model.layers.12.mlp.experts.128.down_proj", "model.layers.12.mlp.experts.129.down_proj", "model.layers.12.mlp.experts.130.down_proj", "model.layers.12.mlp.experts.131.down_proj", "model.layers.12.mlp.experts.132.down_proj", "model.layers.12.mlp.experts.133.down_proj", "model.layers.12.mlp.experts.134.down_proj", "model.layers.12.mlp.experts.135.down_proj", "model.layers.12.mlp.experts.136.down_proj", "model.layers.12.mlp.experts.137.down_proj", "model.layers.12.mlp.experts.138.down_proj", "model.layers.12.mlp.experts.139.down_proj", "model.layers.12.mlp.experts.140.down_proj", "model.layers.12.mlp.experts.141.down_proj", "model.layers.12.mlp.experts.142.down_proj", "model.layers.12.mlp.experts.143.down_proj", "model.layers.12.mlp.experts.144.down_proj", "model.layers.12.mlp.experts.145.down_proj", "model.layers.12.mlp.experts.146.down_proj", "model.layers.12.mlp.experts.147.down_proj", "model.layers.12.mlp.experts.148.down_proj", "model.layers.12.mlp.experts.149.down_proj", "model.layers.12.mlp.experts.150.down_proj", "model.layers.12.mlp.experts.151.down_proj", "model.layers.12.mlp.experts.152.down_proj", "model.layers.12.mlp.experts.153.down_proj", "model.layers.12.mlp.experts.154.down_proj", "model.layers.12.mlp.experts.155.down_proj", "model.layers.12.mlp.experts.156.down_proj", "model.layers.12.mlp.experts.157.down_proj", "model.layers.12.mlp.experts.158.down_proj", "model.layers.12.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.874951703939587e-05, "dbits": 2516582400 } ] }, { "idx": 75, "layers": [ "model.layers.13.self_attn.q_proj" ], "candidates": [ { "dkld": 1.676195533946152e-05, "dbits": 125829120 } ] }, { "idx": 76, "layers": [ "model.layers.13.self_attn.k_proj", "model.layers.13.self_attn.v_proj" ], "candidates": [ { "dkld": -1.432969584129755e-05, "dbits": 20971520 } ] }, { "idx": 77, "layers": [ "model.layers.13.self_attn.o_proj" ], "candidates": [ { "dkld": -3.951878170482814e-06, "dbits": 125829120 } ] }, { "idx": 78, "layers": [ "model.layers.13.mlp.shared_experts.gate_proj", "model.layers.13.mlp.shared_experts.up_proj", "model.layers.13.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.704706513322977e-06, "dbits": 47185920 } ] }, { "idx": 79, "layers": [ "model.layers.13.mlp.experts.0.gate_proj", "model.layers.13.mlp.experts.1.gate_proj", "model.layers.13.mlp.experts.2.gate_proj", "model.layers.13.mlp.experts.3.gate_proj", "model.layers.13.mlp.experts.4.gate_proj", "model.layers.13.mlp.experts.5.gate_proj", "model.layers.13.mlp.experts.6.gate_proj", "model.layers.13.mlp.experts.7.gate_proj", "model.layers.13.mlp.experts.8.gate_proj", "model.layers.13.mlp.experts.9.gate_proj", "model.layers.13.mlp.experts.10.gate_proj", "model.layers.13.mlp.experts.11.gate_proj", "model.layers.13.mlp.experts.12.gate_proj", "model.layers.13.mlp.experts.13.gate_proj", "model.layers.13.mlp.experts.14.gate_proj", "model.layers.13.mlp.experts.15.gate_proj", "model.layers.13.mlp.experts.16.gate_proj", "model.layers.13.mlp.experts.17.gate_proj", "model.layers.13.mlp.experts.18.gate_proj", "model.layers.13.mlp.experts.19.gate_proj", "model.layers.13.mlp.experts.20.gate_proj", "model.layers.13.mlp.experts.21.gate_proj", "model.layers.13.mlp.experts.22.gate_proj", "model.layers.13.mlp.experts.23.gate_proj", "model.layers.13.mlp.experts.24.gate_proj", "model.layers.13.mlp.experts.25.gate_proj", "model.layers.13.mlp.experts.26.gate_proj", "model.layers.13.mlp.experts.27.gate_proj", "model.layers.13.mlp.experts.28.gate_proj", "model.layers.13.mlp.experts.29.gate_proj", "model.layers.13.mlp.experts.30.gate_proj", "model.layers.13.mlp.experts.31.gate_proj", "model.layers.13.mlp.experts.32.gate_proj", "model.layers.13.mlp.experts.33.gate_proj", "model.layers.13.mlp.experts.34.gate_proj", "model.layers.13.mlp.experts.35.gate_proj", "model.layers.13.mlp.experts.36.gate_proj", "model.layers.13.mlp.experts.37.gate_proj", "model.layers.13.mlp.experts.38.gate_proj", "model.layers.13.mlp.experts.39.gate_proj", "model.layers.13.mlp.experts.40.gate_proj", "model.layers.13.mlp.experts.41.gate_proj", "model.layers.13.mlp.experts.42.gate_proj", "model.layers.13.mlp.experts.43.gate_proj", "model.layers.13.mlp.experts.44.gate_proj", "model.layers.13.mlp.experts.45.gate_proj", "model.layers.13.mlp.experts.46.gate_proj", "model.layers.13.mlp.experts.47.gate_proj", "model.layers.13.mlp.experts.48.gate_proj", "model.layers.13.mlp.experts.49.gate_proj", "model.layers.13.mlp.experts.50.gate_proj", "model.layers.13.mlp.experts.51.gate_proj", "model.layers.13.mlp.experts.52.gate_proj", "model.layers.13.mlp.experts.53.gate_proj", "model.layers.13.mlp.experts.54.gate_proj", "model.layers.13.mlp.experts.55.gate_proj", "model.layers.13.mlp.experts.56.gate_proj", "model.layers.13.mlp.experts.57.gate_proj", "model.layers.13.mlp.experts.58.gate_proj", "model.layers.13.mlp.experts.59.gate_proj", "model.layers.13.mlp.experts.60.gate_proj", "model.layers.13.mlp.experts.61.gate_proj", "model.layers.13.mlp.experts.62.gate_proj", "model.layers.13.mlp.experts.63.gate_proj", "model.layers.13.mlp.experts.64.gate_proj", "model.layers.13.mlp.experts.65.gate_proj", "model.layers.13.mlp.experts.66.gate_proj", "model.layers.13.mlp.experts.67.gate_proj", "model.layers.13.mlp.experts.68.gate_proj", "model.layers.13.mlp.experts.69.gate_proj", "model.layers.13.mlp.experts.70.gate_proj", "model.layers.13.mlp.experts.71.gate_proj", "model.layers.13.mlp.experts.72.gate_proj", "model.layers.13.mlp.experts.73.gate_proj", "model.layers.13.mlp.experts.74.gate_proj", "model.layers.13.mlp.experts.75.gate_proj", "model.layers.13.mlp.experts.76.gate_proj", "model.layers.13.mlp.experts.77.gate_proj", "model.layers.13.mlp.experts.78.gate_proj", "model.layers.13.mlp.experts.79.gate_proj", "model.layers.13.mlp.experts.80.gate_proj", "model.layers.13.mlp.experts.81.gate_proj", "model.layers.13.mlp.experts.82.gate_proj", "model.layers.13.mlp.experts.83.gate_proj", "model.layers.13.mlp.experts.84.gate_proj", "model.layers.13.mlp.experts.85.gate_proj", "model.layers.13.mlp.experts.86.gate_proj", "model.layers.13.mlp.experts.87.gate_proj", "model.layers.13.mlp.experts.88.gate_proj", "model.layers.13.mlp.experts.89.gate_proj", "model.layers.13.mlp.experts.90.gate_proj", "model.layers.13.mlp.experts.91.gate_proj", "model.layers.13.mlp.experts.92.gate_proj", "model.layers.13.mlp.experts.93.gate_proj", "model.layers.13.mlp.experts.94.gate_proj", "model.layers.13.mlp.experts.95.gate_proj", "model.layers.13.mlp.experts.96.gate_proj", "model.layers.13.mlp.experts.97.gate_proj", "model.layers.13.mlp.experts.98.gate_proj", "model.layers.13.mlp.experts.99.gate_proj", "model.layers.13.mlp.experts.100.gate_proj", "model.layers.13.mlp.experts.101.gate_proj", "model.layers.13.mlp.experts.102.gate_proj", "model.layers.13.mlp.experts.103.gate_proj", "model.layers.13.mlp.experts.104.gate_proj", "model.layers.13.mlp.experts.105.gate_proj", "model.layers.13.mlp.experts.106.gate_proj", "model.layers.13.mlp.experts.107.gate_proj", "model.layers.13.mlp.experts.108.gate_proj", "model.layers.13.mlp.experts.109.gate_proj", "model.layers.13.mlp.experts.110.gate_proj", "model.layers.13.mlp.experts.111.gate_proj", "model.layers.13.mlp.experts.112.gate_proj", "model.layers.13.mlp.experts.113.gate_proj", "model.layers.13.mlp.experts.114.gate_proj", "model.layers.13.mlp.experts.115.gate_proj", "model.layers.13.mlp.experts.116.gate_proj", "model.layers.13.mlp.experts.117.gate_proj", "model.layers.13.mlp.experts.118.gate_proj", "model.layers.13.mlp.experts.119.gate_proj", "model.layers.13.mlp.experts.120.gate_proj", "model.layers.13.mlp.experts.121.gate_proj", "model.layers.13.mlp.experts.122.gate_proj", "model.layers.13.mlp.experts.123.gate_proj", "model.layers.13.mlp.experts.124.gate_proj", "model.layers.13.mlp.experts.125.gate_proj", "model.layers.13.mlp.experts.126.gate_proj", "model.layers.13.mlp.experts.127.gate_proj", "model.layers.13.mlp.experts.128.gate_proj", "model.layers.13.mlp.experts.129.gate_proj", "model.layers.13.mlp.experts.130.gate_proj", "model.layers.13.mlp.experts.131.gate_proj", "model.layers.13.mlp.experts.132.gate_proj", "model.layers.13.mlp.experts.133.gate_proj", "model.layers.13.mlp.experts.134.gate_proj", "model.layers.13.mlp.experts.135.gate_proj", "model.layers.13.mlp.experts.136.gate_proj", "model.layers.13.mlp.experts.137.gate_proj", "model.layers.13.mlp.experts.138.gate_proj", "model.layers.13.mlp.experts.139.gate_proj", "model.layers.13.mlp.experts.140.gate_proj", "model.layers.13.mlp.experts.141.gate_proj", "model.layers.13.mlp.experts.142.gate_proj", "model.layers.13.mlp.experts.143.gate_proj", "model.layers.13.mlp.experts.144.gate_proj", "model.layers.13.mlp.experts.145.gate_proj", "model.layers.13.mlp.experts.146.gate_proj", "model.layers.13.mlp.experts.147.gate_proj", "model.layers.13.mlp.experts.148.gate_proj", "model.layers.13.mlp.experts.149.gate_proj", "model.layers.13.mlp.experts.150.gate_proj", "model.layers.13.mlp.experts.151.gate_proj", "model.layers.13.mlp.experts.152.gate_proj", "model.layers.13.mlp.experts.153.gate_proj", "model.layers.13.mlp.experts.154.gate_proj", "model.layers.13.mlp.experts.155.gate_proj", "model.layers.13.mlp.experts.156.gate_proj", "model.layers.13.mlp.experts.157.gate_proj", "model.layers.13.mlp.experts.158.gate_proj", "model.layers.13.mlp.experts.159.gate_proj", "model.layers.13.mlp.experts.0.up_proj", "model.layers.13.mlp.experts.1.up_proj", "model.layers.13.mlp.experts.2.up_proj", "model.layers.13.mlp.experts.3.up_proj", "model.layers.13.mlp.experts.4.up_proj", "model.layers.13.mlp.experts.5.up_proj", "model.layers.13.mlp.experts.6.up_proj", "model.layers.13.mlp.experts.7.up_proj", "model.layers.13.mlp.experts.8.up_proj", "model.layers.13.mlp.experts.9.up_proj", "model.layers.13.mlp.experts.10.up_proj", "model.layers.13.mlp.experts.11.up_proj", "model.layers.13.mlp.experts.12.up_proj", "model.layers.13.mlp.experts.13.up_proj", "model.layers.13.mlp.experts.14.up_proj", "model.layers.13.mlp.experts.15.up_proj", "model.layers.13.mlp.experts.16.up_proj", "model.layers.13.mlp.experts.17.up_proj", "model.layers.13.mlp.experts.18.up_proj", "model.layers.13.mlp.experts.19.up_proj", "model.layers.13.mlp.experts.20.up_proj", "model.layers.13.mlp.experts.21.up_proj", "model.layers.13.mlp.experts.22.up_proj", "model.layers.13.mlp.experts.23.up_proj", "model.layers.13.mlp.experts.24.up_proj", "model.layers.13.mlp.experts.25.up_proj", "model.layers.13.mlp.experts.26.up_proj", "model.layers.13.mlp.experts.27.up_proj", "model.layers.13.mlp.experts.28.up_proj", "model.layers.13.mlp.experts.29.up_proj", "model.layers.13.mlp.experts.30.up_proj", "model.layers.13.mlp.experts.31.up_proj", "model.layers.13.mlp.experts.32.up_proj", "model.layers.13.mlp.experts.33.up_proj", "model.layers.13.mlp.experts.34.up_proj", "model.layers.13.mlp.experts.35.up_proj", "model.layers.13.mlp.experts.36.up_proj", "model.layers.13.mlp.experts.37.up_proj", "model.layers.13.mlp.experts.38.up_proj", "model.layers.13.mlp.experts.39.up_proj", "model.layers.13.mlp.experts.40.up_proj", "model.layers.13.mlp.experts.41.up_proj", "model.layers.13.mlp.experts.42.up_proj", "model.layers.13.mlp.experts.43.up_proj", "model.layers.13.mlp.experts.44.up_proj", "model.layers.13.mlp.experts.45.up_proj", "model.layers.13.mlp.experts.46.up_proj", "model.layers.13.mlp.experts.47.up_proj", "model.layers.13.mlp.experts.48.up_proj", "model.layers.13.mlp.experts.49.up_proj", "model.layers.13.mlp.experts.50.up_proj", "model.layers.13.mlp.experts.51.up_proj", "model.layers.13.mlp.experts.52.up_proj", "model.layers.13.mlp.experts.53.up_proj", "model.layers.13.mlp.experts.54.up_proj", "model.layers.13.mlp.experts.55.up_proj", "model.layers.13.mlp.experts.56.up_proj", "model.layers.13.mlp.experts.57.up_proj", "model.layers.13.mlp.experts.58.up_proj", "model.layers.13.mlp.experts.59.up_proj", "model.layers.13.mlp.experts.60.up_proj", "model.layers.13.mlp.experts.61.up_proj", "model.layers.13.mlp.experts.62.up_proj", "model.layers.13.mlp.experts.63.up_proj", "model.layers.13.mlp.experts.64.up_proj", "model.layers.13.mlp.experts.65.up_proj", "model.layers.13.mlp.experts.66.up_proj", "model.layers.13.mlp.experts.67.up_proj", "model.layers.13.mlp.experts.68.up_proj", "model.layers.13.mlp.experts.69.up_proj", "model.layers.13.mlp.experts.70.up_proj", "model.layers.13.mlp.experts.71.up_proj", "model.layers.13.mlp.experts.72.up_proj", "model.layers.13.mlp.experts.73.up_proj", "model.layers.13.mlp.experts.74.up_proj", "model.layers.13.mlp.experts.75.up_proj", "model.layers.13.mlp.experts.76.up_proj", "model.layers.13.mlp.experts.77.up_proj", "model.layers.13.mlp.experts.78.up_proj", "model.layers.13.mlp.experts.79.up_proj", "model.layers.13.mlp.experts.80.up_proj", "model.layers.13.mlp.experts.81.up_proj", "model.layers.13.mlp.experts.82.up_proj", "model.layers.13.mlp.experts.83.up_proj", "model.layers.13.mlp.experts.84.up_proj", "model.layers.13.mlp.experts.85.up_proj", "model.layers.13.mlp.experts.86.up_proj", "model.layers.13.mlp.experts.87.up_proj", "model.layers.13.mlp.experts.88.up_proj", "model.layers.13.mlp.experts.89.up_proj", "model.layers.13.mlp.experts.90.up_proj", "model.layers.13.mlp.experts.91.up_proj", "model.layers.13.mlp.experts.92.up_proj", "model.layers.13.mlp.experts.93.up_proj", "model.layers.13.mlp.experts.94.up_proj", "model.layers.13.mlp.experts.95.up_proj", "model.layers.13.mlp.experts.96.up_proj", "model.layers.13.mlp.experts.97.up_proj", "model.layers.13.mlp.experts.98.up_proj", "model.layers.13.mlp.experts.99.up_proj", "model.layers.13.mlp.experts.100.up_proj", "model.layers.13.mlp.experts.101.up_proj", "model.layers.13.mlp.experts.102.up_proj", "model.layers.13.mlp.experts.103.up_proj", "model.layers.13.mlp.experts.104.up_proj", "model.layers.13.mlp.experts.105.up_proj", "model.layers.13.mlp.experts.106.up_proj", "model.layers.13.mlp.experts.107.up_proj", "model.layers.13.mlp.experts.108.up_proj", "model.layers.13.mlp.experts.109.up_proj", "model.layers.13.mlp.experts.110.up_proj", "model.layers.13.mlp.experts.111.up_proj", "model.layers.13.mlp.experts.112.up_proj", "model.layers.13.mlp.experts.113.up_proj", "model.layers.13.mlp.experts.114.up_proj", "model.layers.13.mlp.experts.115.up_proj", "model.layers.13.mlp.experts.116.up_proj", "model.layers.13.mlp.experts.117.up_proj", "model.layers.13.mlp.experts.118.up_proj", "model.layers.13.mlp.experts.119.up_proj", "model.layers.13.mlp.experts.120.up_proj", "model.layers.13.mlp.experts.121.up_proj", "model.layers.13.mlp.experts.122.up_proj", "model.layers.13.mlp.experts.123.up_proj", "model.layers.13.mlp.experts.124.up_proj", "model.layers.13.mlp.experts.125.up_proj", "model.layers.13.mlp.experts.126.up_proj", "model.layers.13.mlp.experts.127.up_proj", "model.layers.13.mlp.experts.128.up_proj", "model.layers.13.mlp.experts.129.up_proj", "model.layers.13.mlp.experts.130.up_proj", "model.layers.13.mlp.experts.131.up_proj", "model.layers.13.mlp.experts.132.up_proj", "model.layers.13.mlp.experts.133.up_proj", "model.layers.13.mlp.experts.134.up_proj", "model.layers.13.mlp.experts.135.up_proj", "model.layers.13.mlp.experts.136.up_proj", "model.layers.13.mlp.experts.137.up_proj", "model.layers.13.mlp.experts.138.up_proj", "model.layers.13.mlp.experts.139.up_proj", "model.layers.13.mlp.experts.140.up_proj", "model.layers.13.mlp.experts.141.up_proj", "model.layers.13.mlp.experts.142.up_proj", "model.layers.13.mlp.experts.143.up_proj", "model.layers.13.mlp.experts.144.up_proj", "model.layers.13.mlp.experts.145.up_proj", "model.layers.13.mlp.experts.146.up_proj", "model.layers.13.mlp.experts.147.up_proj", "model.layers.13.mlp.experts.148.up_proj", "model.layers.13.mlp.experts.149.up_proj", "model.layers.13.mlp.experts.150.up_proj", "model.layers.13.mlp.experts.151.up_proj", "model.layers.13.mlp.experts.152.up_proj", "model.layers.13.mlp.experts.153.up_proj", "model.layers.13.mlp.experts.154.up_proj", "model.layers.13.mlp.experts.155.up_proj", "model.layers.13.mlp.experts.156.up_proj", "model.layers.13.mlp.experts.157.up_proj", "model.layers.13.mlp.experts.158.up_proj", "model.layers.13.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.328750899527226e-05, "dbits": 5033164800 } ] }, { "idx": 80, "layers": [ "model.layers.13.mlp.experts.0.down_proj", "model.layers.13.mlp.experts.1.down_proj", "model.layers.13.mlp.experts.2.down_proj", "model.layers.13.mlp.experts.3.down_proj", "model.layers.13.mlp.experts.4.down_proj", "model.layers.13.mlp.experts.5.down_proj", "model.layers.13.mlp.experts.6.down_proj", "model.layers.13.mlp.experts.7.down_proj", "model.layers.13.mlp.experts.8.down_proj", "model.layers.13.mlp.experts.9.down_proj", "model.layers.13.mlp.experts.10.down_proj", "model.layers.13.mlp.experts.11.down_proj", "model.layers.13.mlp.experts.12.down_proj", "model.layers.13.mlp.experts.13.down_proj", "model.layers.13.mlp.experts.14.down_proj", "model.layers.13.mlp.experts.15.down_proj", "model.layers.13.mlp.experts.16.down_proj", "model.layers.13.mlp.experts.17.down_proj", "model.layers.13.mlp.experts.18.down_proj", "model.layers.13.mlp.experts.19.down_proj", "model.layers.13.mlp.experts.20.down_proj", "model.layers.13.mlp.experts.21.down_proj", "model.layers.13.mlp.experts.22.down_proj", "model.layers.13.mlp.experts.23.down_proj", "model.layers.13.mlp.experts.24.down_proj", "model.layers.13.mlp.experts.25.down_proj", "model.layers.13.mlp.experts.26.down_proj", "model.layers.13.mlp.experts.27.down_proj", "model.layers.13.mlp.experts.28.down_proj", "model.layers.13.mlp.experts.29.down_proj", "model.layers.13.mlp.experts.30.down_proj", "model.layers.13.mlp.experts.31.down_proj", "model.layers.13.mlp.experts.32.down_proj", "model.layers.13.mlp.experts.33.down_proj", "model.layers.13.mlp.experts.34.down_proj", "model.layers.13.mlp.experts.35.down_proj", "model.layers.13.mlp.experts.36.down_proj", "model.layers.13.mlp.experts.37.down_proj", "model.layers.13.mlp.experts.38.down_proj", "model.layers.13.mlp.experts.39.down_proj", "model.layers.13.mlp.experts.40.down_proj", "model.layers.13.mlp.experts.41.down_proj", "model.layers.13.mlp.experts.42.down_proj", "model.layers.13.mlp.experts.43.down_proj", "model.layers.13.mlp.experts.44.down_proj", "model.layers.13.mlp.experts.45.down_proj", "model.layers.13.mlp.experts.46.down_proj", "model.layers.13.mlp.experts.47.down_proj", "model.layers.13.mlp.experts.48.down_proj", "model.layers.13.mlp.experts.49.down_proj", "model.layers.13.mlp.experts.50.down_proj", "model.layers.13.mlp.experts.51.down_proj", "model.layers.13.mlp.experts.52.down_proj", "model.layers.13.mlp.experts.53.down_proj", "model.layers.13.mlp.experts.54.down_proj", "model.layers.13.mlp.experts.55.down_proj", "model.layers.13.mlp.experts.56.down_proj", "model.layers.13.mlp.experts.57.down_proj", "model.layers.13.mlp.experts.58.down_proj", "model.layers.13.mlp.experts.59.down_proj", "model.layers.13.mlp.experts.60.down_proj", "model.layers.13.mlp.experts.61.down_proj", "model.layers.13.mlp.experts.62.down_proj", "model.layers.13.mlp.experts.63.down_proj", "model.layers.13.mlp.experts.64.down_proj", "model.layers.13.mlp.experts.65.down_proj", "model.layers.13.mlp.experts.66.down_proj", "model.layers.13.mlp.experts.67.down_proj", "model.layers.13.mlp.experts.68.down_proj", "model.layers.13.mlp.experts.69.down_proj", "model.layers.13.mlp.experts.70.down_proj", "model.layers.13.mlp.experts.71.down_proj", "model.layers.13.mlp.experts.72.down_proj", "model.layers.13.mlp.experts.73.down_proj", "model.layers.13.mlp.experts.74.down_proj", "model.layers.13.mlp.experts.75.down_proj", "model.layers.13.mlp.experts.76.down_proj", "model.layers.13.mlp.experts.77.down_proj", "model.layers.13.mlp.experts.78.down_proj", "model.layers.13.mlp.experts.79.down_proj", "model.layers.13.mlp.experts.80.down_proj", "model.layers.13.mlp.experts.81.down_proj", "model.layers.13.mlp.experts.82.down_proj", "model.layers.13.mlp.experts.83.down_proj", "model.layers.13.mlp.experts.84.down_proj", "model.layers.13.mlp.experts.85.down_proj", "model.layers.13.mlp.experts.86.down_proj", "model.layers.13.mlp.experts.87.down_proj", "model.layers.13.mlp.experts.88.down_proj", "model.layers.13.mlp.experts.89.down_proj", "model.layers.13.mlp.experts.90.down_proj", "model.layers.13.mlp.experts.91.down_proj", "model.layers.13.mlp.experts.92.down_proj", "model.layers.13.mlp.experts.93.down_proj", "model.layers.13.mlp.experts.94.down_proj", "model.layers.13.mlp.experts.95.down_proj", "model.layers.13.mlp.experts.96.down_proj", "model.layers.13.mlp.experts.97.down_proj", "model.layers.13.mlp.experts.98.down_proj", "model.layers.13.mlp.experts.99.down_proj", "model.layers.13.mlp.experts.100.down_proj", "model.layers.13.mlp.experts.101.down_proj", "model.layers.13.mlp.experts.102.down_proj", "model.layers.13.mlp.experts.103.down_proj", "model.layers.13.mlp.experts.104.down_proj", "model.layers.13.mlp.experts.105.down_proj", "model.layers.13.mlp.experts.106.down_proj", "model.layers.13.mlp.experts.107.down_proj", "model.layers.13.mlp.experts.108.down_proj", "model.layers.13.mlp.experts.109.down_proj", "model.layers.13.mlp.experts.110.down_proj", "model.layers.13.mlp.experts.111.down_proj", "model.layers.13.mlp.experts.112.down_proj", "model.layers.13.mlp.experts.113.down_proj", "model.layers.13.mlp.experts.114.down_proj", "model.layers.13.mlp.experts.115.down_proj", "model.layers.13.mlp.experts.116.down_proj", "model.layers.13.mlp.experts.117.down_proj", "model.layers.13.mlp.experts.118.down_proj", "model.layers.13.mlp.experts.119.down_proj", "model.layers.13.mlp.experts.120.down_proj", "model.layers.13.mlp.experts.121.down_proj", "model.layers.13.mlp.experts.122.down_proj", "model.layers.13.mlp.experts.123.down_proj", "model.layers.13.mlp.experts.124.down_proj", "model.layers.13.mlp.experts.125.down_proj", "model.layers.13.mlp.experts.126.down_proj", "model.layers.13.mlp.experts.127.down_proj", "model.layers.13.mlp.experts.128.down_proj", "model.layers.13.mlp.experts.129.down_proj", "model.layers.13.mlp.experts.130.down_proj", "model.layers.13.mlp.experts.131.down_proj", "model.layers.13.mlp.experts.132.down_proj", "model.layers.13.mlp.experts.133.down_proj", "model.layers.13.mlp.experts.134.down_proj", "model.layers.13.mlp.experts.135.down_proj", "model.layers.13.mlp.experts.136.down_proj", "model.layers.13.mlp.experts.137.down_proj", "model.layers.13.mlp.experts.138.down_proj", "model.layers.13.mlp.experts.139.down_proj", "model.layers.13.mlp.experts.140.down_proj", "model.layers.13.mlp.experts.141.down_proj", "model.layers.13.mlp.experts.142.down_proj", "model.layers.13.mlp.experts.143.down_proj", "model.layers.13.mlp.experts.144.down_proj", "model.layers.13.mlp.experts.145.down_proj", "model.layers.13.mlp.experts.146.down_proj", "model.layers.13.mlp.experts.147.down_proj", "model.layers.13.mlp.experts.148.down_proj", "model.layers.13.mlp.experts.149.down_proj", "model.layers.13.mlp.experts.150.down_proj", "model.layers.13.mlp.experts.151.down_proj", "model.layers.13.mlp.experts.152.down_proj", "model.layers.13.mlp.experts.153.down_proj", "model.layers.13.mlp.experts.154.down_proj", "model.layers.13.mlp.experts.155.down_proj", "model.layers.13.mlp.experts.156.down_proj", "model.layers.13.mlp.experts.157.down_proj", "model.layers.13.mlp.experts.158.down_proj", "model.layers.13.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.7113253124989526e-05, "dbits": 2516582400 } ] }, { "idx": 81, "layers": [ "model.layers.14.self_attn.q_proj" ], "candidates": [ { "dkld": 9.760761167854157e-06, "dbits": 125829120 } ] }, { "idx": 82, "layers": [ "model.layers.14.self_attn.k_proj", "model.layers.14.self_attn.v_proj" ], "candidates": [ { "dkld": -9.71422705333682e-06, "dbits": 20971520 } ] }, { "idx": 83, "layers": [ "model.layers.14.self_attn.o_proj" ], "candidates": [ { "dkld": -8.872195030562601e-06, "dbits": 125829120 } ] }, { "idx": 84, "layers": [ "model.layers.14.mlp.shared_experts.gate_proj", "model.layers.14.mlp.shared_experts.up_proj", "model.layers.14.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.897351467050612e-06, "dbits": 47185920 } ] }, { "idx": 85, "layers": [ "model.layers.14.mlp.experts.0.gate_proj", "model.layers.14.mlp.experts.1.gate_proj", "model.layers.14.mlp.experts.2.gate_proj", "model.layers.14.mlp.experts.3.gate_proj", "model.layers.14.mlp.experts.4.gate_proj", "model.layers.14.mlp.experts.5.gate_proj", "model.layers.14.mlp.experts.6.gate_proj", "model.layers.14.mlp.experts.7.gate_proj", "model.layers.14.mlp.experts.8.gate_proj", "model.layers.14.mlp.experts.9.gate_proj", "model.layers.14.mlp.experts.10.gate_proj", "model.layers.14.mlp.experts.11.gate_proj", "model.layers.14.mlp.experts.12.gate_proj", "model.layers.14.mlp.experts.13.gate_proj", "model.layers.14.mlp.experts.14.gate_proj", "model.layers.14.mlp.experts.15.gate_proj", "model.layers.14.mlp.experts.16.gate_proj", "model.layers.14.mlp.experts.17.gate_proj", "model.layers.14.mlp.experts.18.gate_proj", "model.layers.14.mlp.experts.19.gate_proj", "model.layers.14.mlp.experts.20.gate_proj", "model.layers.14.mlp.experts.21.gate_proj", "model.layers.14.mlp.experts.22.gate_proj", "model.layers.14.mlp.experts.23.gate_proj", "model.layers.14.mlp.experts.24.gate_proj", "model.layers.14.mlp.experts.25.gate_proj", "model.layers.14.mlp.experts.26.gate_proj", "model.layers.14.mlp.experts.27.gate_proj", "model.layers.14.mlp.experts.28.gate_proj", "model.layers.14.mlp.experts.29.gate_proj", "model.layers.14.mlp.experts.30.gate_proj", "model.layers.14.mlp.experts.31.gate_proj", "model.layers.14.mlp.experts.32.gate_proj", "model.layers.14.mlp.experts.33.gate_proj", "model.layers.14.mlp.experts.34.gate_proj", "model.layers.14.mlp.experts.35.gate_proj", "model.layers.14.mlp.experts.36.gate_proj", "model.layers.14.mlp.experts.37.gate_proj", "model.layers.14.mlp.experts.38.gate_proj", "model.layers.14.mlp.experts.39.gate_proj", "model.layers.14.mlp.experts.40.gate_proj", "model.layers.14.mlp.experts.41.gate_proj", "model.layers.14.mlp.experts.42.gate_proj", "model.layers.14.mlp.experts.43.gate_proj", "model.layers.14.mlp.experts.44.gate_proj", "model.layers.14.mlp.experts.45.gate_proj", "model.layers.14.mlp.experts.46.gate_proj", "model.layers.14.mlp.experts.47.gate_proj", "model.layers.14.mlp.experts.48.gate_proj", "model.layers.14.mlp.experts.49.gate_proj", "model.layers.14.mlp.experts.50.gate_proj", "model.layers.14.mlp.experts.51.gate_proj", "model.layers.14.mlp.experts.52.gate_proj", "model.layers.14.mlp.experts.53.gate_proj", "model.layers.14.mlp.experts.54.gate_proj", "model.layers.14.mlp.experts.55.gate_proj", "model.layers.14.mlp.experts.56.gate_proj", "model.layers.14.mlp.experts.57.gate_proj", "model.layers.14.mlp.experts.58.gate_proj", "model.layers.14.mlp.experts.59.gate_proj", "model.layers.14.mlp.experts.60.gate_proj", "model.layers.14.mlp.experts.61.gate_proj", "model.layers.14.mlp.experts.62.gate_proj", "model.layers.14.mlp.experts.63.gate_proj", "model.layers.14.mlp.experts.64.gate_proj", "model.layers.14.mlp.experts.65.gate_proj", "model.layers.14.mlp.experts.66.gate_proj", "model.layers.14.mlp.experts.67.gate_proj", "model.layers.14.mlp.experts.68.gate_proj", "model.layers.14.mlp.experts.69.gate_proj", "model.layers.14.mlp.experts.70.gate_proj", "model.layers.14.mlp.experts.71.gate_proj", "model.layers.14.mlp.experts.72.gate_proj", "model.layers.14.mlp.experts.73.gate_proj", "model.layers.14.mlp.experts.74.gate_proj", "model.layers.14.mlp.experts.75.gate_proj", "model.layers.14.mlp.experts.76.gate_proj", "model.layers.14.mlp.experts.77.gate_proj", "model.layers.14.mlp.experts.78.gate_proj", "model.layers.14.mlp.experts.79.gate_proj", "model.layers.14.mlp.experts.80.gate_proj", "model.layers.14.mlp.experts.81.gate_proj", "model.layers.14.mlp.experts.82.gate_proj", "model.layers.14.mlp.experts.83.gate_proj", "model.layers.14.mlp.experts.84.gate_proj", "model.layers.14.mlp.experts.85.gate_proj", "model.layers.14.mlp.experts.86.gate_proj", "model.layers.14.mlp.experts.87.gate_proj", "model.layers.14.mlp.experts.88.gate_proj", "model.layers.14.mlp.experts.89.gate_proj", "model.layers.14.mlp.experts.90.gate_proj", "model.layers.14.mlp.experts.91.gate_proj", "model.layers.14.mlp.experts.92.gate_proj", "model.layers.14.mlp.experts.93.gate_proj", "model.layers.14.mlp.experts.94.gate_proj", "model.layers.14.mlp.experts.95.gate_proj", "model.layers.14.mlp.experts.96.gate_proj", "model.layers.14.mlp.experts.97.gate_proj", "model.layers.14.mlp.experts.98.gate_proj", "model.layers.14.mlp.experts.99.gate_proj", "model.layers.14.mlp.experts.100.gate_proj", "model.layers.14.mlp.experts.101.gate_proj", "model.layers.14.mlp.experts.102.gate_proj", "model.layers.14.mlp.experts.103.gate_proj", "model.layers.14.mlp.experts.104.gate_proj", "model.layers.14.mlp.experts.105.gate_proj", "model.layers.14.mlp.experts.106.gate_proj", "model.layers.14.mlp.experts.107.gate_proj", "model.layers.14.mlp.experts.108.gate_proj", "model.layers.14.mlp.experts.109.gate_proj", "model.layers.14.mlp.experts.110.gate_proj", "model.layers.14.mlp.experts.111.gate_proj", "model.layers.14.mlp.experts.112.gate_proj", "model.layers.14.mlp.experts.113.gate_proj", "model.layers.14.mlp.experts.114.gate_proj", "model.layers.14.mlp.experts.115.gate_proj", "model.layers.14.mlp.experts.116.gate_proj", "model.layers.14.mlp.experts.117.gate_proj", "model.layers.14.mlp.experts.118.gate_proj", "model.layers.14.mlp.experts.119.gate_proj", "model.layers.14.mlp.experts.120.gate_proj", "model.layers.14.mlp.experts.121.gate_proj", "model.layers.14.mlp.experts.122.gate_proj", "model.layers.14.mlp.experts.123.gate_proj", "model.layers.14.mlp.experts.124.gate_proj", "model.layers.14.mlp.experts.125.gate_proj", "model.layers.14.mlp.experts.126.gate_proj", "model.layers.14.mlp.experts.127.gate_proj", "model.layers.14.mlp.experts.128.gate_proj", "model.layers.14.mlp.experts.129.gate_proj", "model.layers.14.mlp.experts.130.gate_proj", "model.layers.14.mlp.experts.131.gate_proj", "model.layers.14.mlp.experts.132.gate_proj", "model.layers.14.mlp.experts.133.gate_proj", "model.layers.14.mlp.experts.134.gate_proj", "model.layers.14.mlp.experts.135.gate_proj", "model.layers.14.mlp.experts.136.gate_proj", "model.layers.14.mlp.experts.137.gate_proj", "model.layers.14.mlp.experts.138.gate_proj", "model.layers.14.mlp.experts.139.gate_proj", "model.layers.14.mlp.experts.140.gate_proj", "model.layers.14.mlp.experts.141.gate_proj", "model.layers.14.mlp.experts.142.gate_proj", "model.layers.14.mlp.experts.143.gate_proj", "model.layers.14.mlp.experts.144.gate_proj", "model.layers.14.mlp.experts.145.gate_proj", "model.layers.14.mlp.experts.146.gate_proj", "model.layers.14.mlp.experts.147.gate_proj", "model.layers.14.mlp.experts.148.gate_proj", "model.layers.14.mlp.experts.149.gate_proj", "model.layers.14.mlp.experts.150.gate_proj", "model.layers.14.mlp.experts.151.gate_proj", "model.layers.14.mlp.experts.152.gate_proj", "model.layers.14.mlp.experts.153.gate_proj", "model.layers.14.mlp.experts.154.gate_proj", "model.layers.14.mlp.experts.155.gate_proj", "model.layers.14.mlp.experts.156.gate_proj", "model.layers.14.mlp.experts.157.gate_proj", "model.layers.14.mlp.experts.158.gate_proj", "model.layers.14.mlp.experts.159.gate_proj", "model.layers.14.mlp.experts.0.up_proj", "model.layers.14.mlp.experts.1.up_proj", "model.layers.14.mlp.experts.2.up_proj", "model.layers.14.mlp.experts.3.up_proj", "model.layers.14.mlp.experts.4.up_proj", "model.layers.14.mlp.experts.5.up_proj", "model.layers.14.mlp.experts.6.up_proj", "model.layers.14.mlp.experts.7.up_proj", "model.layers.14.mlp.experts.8.up_proj", "model.layers.14.mlp.experts.9.up_proj", "model.layers.14.mlp.experts.10.up_proj", "model.layers.14.mlp.experts.11.up_proj", "model.layers.14.mlp.experts.12.up_proj", "model.layers.14.mlp.experts.13.up_proj", "model.layers.14.mlp.experts.14.up_proj", "model.layers.14.mlp.experts.15.up_proj", "model.layers.14.mlp.experts.16.up_proj", "model.layers.14.mlp.experts.17.up_proj", "model.layers.14.mlp.experts.18.up_proj", "model.layers.14.mlp.experts.19.up_proj", "model.layers.14.mlp.experts.20.up_proj", "model.layers.14.mlp.experts.21.up_proj", "model.layers.14.mlp.experts.22.up_proj", "model.layers.14.mlp.experts.23.up_proj", "model.layers.14.mlp.experts.24.up_proj", "model.layers.14.mlp.experts.25.up_proj", "model.layers.14.mlp.experts.26.up_proj", "model.layers.14.mlp.experts.27.up_proj", "model.layers.14.mlp.experts.28.up_proj", "model.layers.14.mlp.experts.29.up_proj", "model.layers.14.mlp.experts.30.up_proj", "model.layers.14.mlp.experts.31.up_proj", "model.layers.14.mlp.experts.32.up_proj", "model.layers.14.mlp.experts.33.up_proj", "model.layers.14.mlp.experts.34.up_proj", "model.layers.14.mlp.experts.35.up_proj", "model.layers.14.mlp.experts.36.up_proj", "model.layers.14.mlp.experts.37.up_proj", "model.layers.14.mlp.experts.38.up_proj", "model.layers.14.mlp.experts.39.up_proj", "model.layers.14.mlp.experts.40.up_proj", "model.layers.14.mlp.experts.41.up_proj", "model.layers.14.mlp.experts.42.up_proj", "model.layers.14.mlp.experts.43.up_proj", "model.layers.14.mlp.experts.44.up_proj", "model.layers.14.mlp.experts.45.up_proj", "model.layers.14.mlp.experts.46.up_proj", "model.layers.14.mlp.experts.47.up_proj", "model.layers.14.mlp.experts.48.up_proj", "model.layers.14.mlp.experts.49.up_proj", "model.layers.14.mlp.experts.50.up_proj", "model.layers.14.mlp.experts.51.up_proj", "model.layers.14.mlp.experts.52.up_proj", "model.layers.14.mlp.experts.53.up_proj", "model.layers.14.mlp.experts.54.up_proj", "model.layers.14.mlp.experts.55.up_proj", "model.layers.14.mlp.experts.56.up_proj", "model.layers.14.mlp.experts.57.up_proj", "model.layers.14.mlp.experts.58.up_proj", "model.layers.14.mlp.experts.59.up_proj", "model.layers.14.mlp.experts.60.up_proj", "model.layers.14.mlp.experts.61.up_proj", "model.layers.14.mlp.experts.62.up_proj", "model.layers.14.mlp.experts.63.up_proj", "model.layers.14.mlp.experts.64.up_proj", "model.layers.14.mlp.experts.65.up_proj", "model.layers.14.mlp.experts.66.up_proj", "model.layers.14.mlp.experts.67.up_proj", "model.layers.14.mlp.experts.68.up_proj", "model.layers.14.mlp.experts.69.up_proj", "model.layers.14.mlp.experts.70.up_proj", "model.layers.14.mlp.experts.71.up_proj", "model.layers.14.mlp.experts.72.up_proj", "model.layers.14.mlp.experts.73.up_proj", "model.layers.14.mlp.experts.74.up_proj", "model.layers.14.mlp.experts.75.up_proj", "model.layers.14.mlp.experts.76.up_proj", "model.layers.14.mlp.experts.77.up_proj", "model.layers.14.mlp.experts.78.up_proj", "model.layers.14.mlp.experts.79.up_proj", "model.layers.14.mlp.experts.80.up_proj", "model.layers.14.mlp.experts.81.up_proj", "model.layers.14.mlp.experts.82.up_proj", "model.layers.14.mlp.experts.83.up_proj", "model.layers.14.mlp.experts.84.up_proj", "model.layers.14.mlp.experts.85.up_proj", "model.layers.14.mlp.experts.86.up_proj", "model.layers.14.mlp.experts.87.up_proj", "model.layers.14.mlp.experts.88.up_proj", "model.layers.14.mlp.experts.89.up_proj", "model.layers.14.mlp.experts.90.up_proj", "model.layers.14.mlp.experts.91.up_proj", "model.layers.14.mlp.experts.92.up_proj", "model.layers.14.mlp.experts.93.up_proj", "model.layers.14.mlp.experts.94.up_proj", "model.layers.14.mlp.experts.95.up_proj", "model.layers.14.mlp.experts.96.up_proj", "model.layers.14.mlp.experts.97.up_proj", "model.layers.14.mlp.experts.98.up_proj", "model.layers.14.mlp.experts.99.up_proj", "model.layers.14.mlp.experts.100.up_proj", "model.layers.14.mlp.experts.101.up_proj", "model.layers.14.mlp.experts.102.up_proj", "model.layers.14.mlp.experts.103.up_proj", "model.layers.14.mlp.experts.104.up_proj", "model.layers.14.mlp.experts.105.up_proj", "model.layers.14.mlp.experts.106.up_proj", "model.layers.14.mlp.experts.107.up_proj", "model.layers.14.mlp.experts.108.up_proj", "model.layers.14.mlp.experts.109.up_proj", "model.layers.14.mlp.experts.110.up_proj", "model.layers.14.mlp.experts.111.up_proj", "model.layers.14.mlp.experts.112.up_proj", "model.layers.14.mlp.experts.113.up_proj", "model.layers.14.mlp.experts.114.up_proj", "model.layers.14.mlp.experts.115.up_proj", "model.layers.14.mlp.experts.116.up_proj", "model.layers.14.mlp.experts.117.up_proj", "model.layers.14.mlp.experts.118.up_proj", "model.layers.14.mlp.experts.119.up_proj", "model.layers.14.mlp.experts.120.up_proj", "model.layers.14.mlp.experts.121.up_proj", "model.layers.14.mlp.experts.122.up_proj", "model.layers.14.mlp.experts.123.up_proj", "model.layers.14.mlp.experts.124.up_proj", "model.layers.14.mlp.experts.125.up_proj", "model.layers.14.mlp.experts.126.up_proj", "model.layers.14.mlp.experts.127.up_proj", "model.layers.14.mlp.experts.128.up_proj", "model.layers.14.mlp.experts.129.up_proj", "model.layers.14.mlp.experts.130.up_proj", "model.layers.14.mlp.experts.131.up_proj", "model.layers.14.mlp.experts.132.up_proj", "model.layers.14.mlp.experts.133.up_proj", "model.layers.14.mlp.experts.134.up_proj", "model.layers.14.mlp.experts.135.up_proj", "model.layers.14.mlp.experts.136.up_proj", "model.layers.14.mlp.experts.137.up_proj", "model.layers.14.mlp.experts.138.up_proj", "model.layers.14.mlp.experts.139.up_proj", "model.layers.14.mlp.experts.140.up_proj", "model.layers.14.mlp.experts.141.up_proj", "model.layers.14.mlp.experts.142.up_proj", "model.layers.14.mlp.experts.143.up_proj", "model.layers.14.mlp.experts.144.up_proj", "model.layers.14.mlp.experts.145.up_proj", "model.layers.14.mlp.experts.146.up_proj", "model.layers.14.mlp.experts.147.up_proj", "model.layers.14.mlp.experts.148.up_proj", "model.layers.14.mlp.experts.149.up_proj", "model.layers.14.mlp.experts.150.up_proj", "model.layers.14.mlp.experts.151.up_proj", "model.layers.14.mlp.experts.152.up_proj", "model.layers.14.mlp.experts.153.up_proj", "model.layers.14.mlp.experts.154.up_proj", "model.layers.14.mlp.experts.155.up_proj", "model.layers.14.mlp.experts.156.up_proj", "model.layers.14.mlp.experts.157.up_proj", "model.layers.14.mlp.experts.158.up_proj", "model.layers.14.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -7.957511115819433e-07, "dbits": 5033164800 } ] }, { "idx": 86, "layers": [ "model.layers.14.mlp.experts.0.down_proj", "model.layers.14.mlp.experts.1.down_proj", "model.layers.14.mlp.experts.2.down_proj", "model.layers.14.mlp.experts.3.down_proj", "model.layers.14.mlp.experts.4.down_proj", "model.layers.14.mlp.experts.5.down_proj", "model.layers.14.mlp.experts.6.down_proj", "model.layers.14.mlp.experts.7.down_proj", "model.layers.14.mlp.experts.8.down_proj", "model.layers.14.mlp.experts.9.down_proj", "model.layers.14.mlp.experts.10.down_proj", "model.layers.14.mlp.experts.11.down_proj", "model.layers.14.mlp.experts.12.down_proj", "model.layers.14.mlp.experts.13.down_proj", "model.layers.14.mlp.experts.14.down_proj", "model.layers.14.mlp.experts.15.down_proj", "model.layers.14.mlp.experts.16.down_proj", "model.layers.14.mlp.experts.17.down_proj", "model.layers.14.mlp.experts.18.down_proj", "model.layers.14.mlp.experts.19.down_proj", "model.layers.14.mlp.experts.20.down_proj", "model.layers.14.mlp.experts.21.down_proj", "model.layers.14.mlp.experts.22.down_proj", "model.layers.14.mlp.experts.23.down_proj", "model.layers.14.mlp.experts.24.down_proj", "model.layers.14.mlp.experts.25.down_proj", "model.layers.14.mlp.experts.26.down_proj", "model.layers.14.mlp.experts.27.down_proj", "model.layers.14.mlp.experts.28.down_proj", "model.layers.14.mlp.experts.29.down_proj", "model.layers.14.mlp.experts.30.down_proj", "model.layers.14.mlp.experts.31.down_proj", "model.layers.14.mlp.experts.32.down_proj", "model.layers.14.mlp.experts.33.down_proj", "model.layers.14.mlp.experts.34.down_proj", "model.layers.14.mlp.experts.35.down_proj", "model.layers.14.mlp.experts.36.down_proj", "model.layers.14.mlp.experts.37.down_proj", "model.layers.14.mlp.experts.38.down_proj", "model.layers.14.mlp.experts.39.down_proj", "model.layers.14.mlp.experts.40.down_proj", "model.layers.14.mlp.experts.41.down_proj", "model.layers.14.mlp.experts.42.down_proj", "model.layers.14.mlp.experts.43.down_proj", "model.layers.14.mlp.experts.44.down_proj", "model.layers.14.mlp.experts.45.down_proj", "model.layers.14.mlp.experts.46.down_proj", "model.layers.14.mlp.experts.47.down_proj", "model.layers.14.mlp.experts.48.down_proj", "model.layers.14.mlp.experts.49.down_proj", "model.layers.14.mlp.experts.50.down_proj", "model.layers.14.mlp.experts.51.down_proj", "model.layers.14.mlp.experts.52.down_proj", "model.layers.14.mlp.experts.53.down_proj", "model.layers.14.mlp.experts.54.down_proj", "model.layers.14.mlp.experts.55.down_proj", "model.layers.14.mlp.experts.56.down_proj", "model.layers.14.mlp.experts.57.down_proj", "model.layers.14.mlp.experts.58.down_proj", "model.layers.14.mlp.experts.59.down_proj", "model.layers.14.mlp.experts.60.down_proj", "model.layers.14.mlp.experts.61.down_proj", "model.layers.14.mlp.experts.62.down_proj", "model.layers.14.mlp.experts.63.down_proj", "model.layers.14.mlp.experts.64.down_proj", "model.layers.14.mlp.experts.65.down_proj", "model.layers.14.mlp.experts.66.down_proj", "model.layers.14.mlp.experts.67.down_proj", "model.layers.14.mlp.experts.68.down_proj", "model.layers.14.mlp.experts.69.down_proj", "model.layers.14.mlp.experts.70.down_proj", "model.layers.14.mlp.experts.71.down_proj", "model.layers.14.mlp.experts.72.down_proj", "model.layers.14.mlp.experts.73.down_proj", "model.layers.14.mlp.experts.74.down_proj", "model.layers.14.mlp.experts.75.down_proj", "model.layers.14.mlp.experts.76.down_proj", "model.layers.14.mlp.experts.77.down_proj", "model.layers.14.mlp.experts.78.down_proj", "model.layers.14.mlp.experts.79.down_proj", "model.layers.14.mlp.experts.80.down_proj", "model.layers.14.mlp.experts.81.down_proj", "model.layers.14.mlp.experts.82.down_proj", "model.layers.14.mlp.experts.83.down_proj", "model.layers.14.mlp.experts.84.down_proj", "model.layers.14.mlp.experts.85.down_proj", "model.layers.14.mlp.experts.86.down_proj", "model.layers.14.mlp.experts.87.down_proj", "model.layers.14.mlp.experts.88.down_proj", "model.layers.14.mlp.experts.89.down_proj", "model.layers.14.mlp.experts.90.down_proj", "model.layers.14.mlp.experts.91.down_proj", "model.layers.14.mlp.experts.92.down_proj", "model.layers.14.mlp.experts.93.down_proj", "model.layers.14.mlp.experts.94.down_proj", "model.layers.14.mlp.experts.95.down_proj", "model.layers.14.mlp.experts.96.down_proj", "model.layers.14.mlp.experts.97.down_proj", "model.layers.14.mlp.experts.98.down_proj", "model.layers.14.mlp.experts.99.down_proj", "model.layers.14.mlp.experts.100.down_proj", "model.layers.14.mlp.experts.101.down_proj", "model.layers.14.mlp.experts.102.down_proj", "model.layers.14.mlp.experts.103.down_proj", "model.layers.14.mlp.experts.104.down_proj", "model.layers.14.mlp.experts.105.down_proj", "model.layers.14.mlp.experts.106.down_proj", "model.layers.14.mlp.experts.107.down_proj", "model.layers.14.mlp.experts.108.down_proj", "model.layers.14.mlp.experts.109.down_proj", "model.layers.14.mlp.experts.110.down_proj", "model.layers.14.mlp.experts.111.down_proj", "model.layers.14.mlp.experts.112.down_proj", "model.layers.14.mlp.experts.113.down_proj", "model.layers.14.mlp.experts.114.down_proj", "model.layers.14.mlp.experts.115.down_proj", "model.layers.14.mlp.experts.116.down_proj", "model.layers.14.mlp.experts.117.down_proj", "model.layers.14.mlp.experts.118.down_proj", "model.layers.14.mlp.experts.119.down_proj", "model.layers.14.mlp.experts.120.down_proj", "model.layers.14.mlp.experts.121.down_proj", "model.layers.14.mlp.experts.122.down_proj", "model.layers.14.mlp.experts.123.down_proj", "model.layers.14.mlp.experts.124.down_proj", "model.layers.14.mlp.experts.125.down_proj", "model.layers.14.mlp.experts.126.down_proj", "model.layers.14.mlp.experts.127.down_proj", "model.layers.14.mlp.experts.128.down_proj", "model.layers.14.mlp.experts.129.down_proj", "model.layers.14.mlp.experts.130.down_proj", "model.layers.14.mlp.experts.131.down_proj", "model.layers.14.mlp.experts.132.down_proj", "model.layers.14.mlp.experts.133.down_proj", "model.layers.14.mlp.experts.134.down_proj", "model.layers.14.mlp.experts.135.down_proj", "model.layers.14.mlp.experts.136.down_proj", "model.layers.14.mlp.experts.137.down_proj", "model.layers.14.mlp.experts.138.down_proj", "model.layers.14.mlp.experts.139.down_proj", "model.layers.14.mlp.experts.140.down_proj", "model.layers.14.mlp.experts.141.down_proj", "model.layers.14.mlp.experts.142.down_proj", "model.layers.14.mlp.experts.143.down_proj", "model.layers.14.mlp.experts.144.down_proj", "model.layers.14.mlp.experts.145.down_proj", "model.layers.14.mlp.experts.146.down_proj", "model.layers.14.mlp.experts.147.down_proj", "model.layers.14.mlp.experts.148.down_proj", "model.layers.14.mlp.experts.149.down_proj", "model.layers.14.mlp.experts.150.down_proj", "model.layers.14.mlp.experts.151.down_proj", "model.layers.14.mlp.experts.152.down_proj", "model.layers.14.mlp.experts.153.down_proj", "model.layers.14.mlp.experts.154.down_proj", "model.layers.14.mlp.experts.155.down_proj", "model.layers.14.mlp.experts.156.down_proj", "model.layers.14.mlp.experts.157.down_proj", "model.layers.14.mlp.experts.158.down_proj", "model.layers.14.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.8124748021364255e-05, "dbits": 2516582400 } ] }, { "idx": 87, "layers": [ "model.layers.15.self_attn.q_proj" ], "candidates": [ { "dkld": -1.095786574296649e-05, "dbits": 125829120 } ] }, { "idx": 88, "layers": [ "model.layers.15.self_attn.k_proj", "model.layers.15.self_attn.v_proj" ], "candidates": [ { "dkld": 5.667115328833504e-06, "dbits": 20971520 } ] }, { "idx": 89, "layers": [ "model.layers.15.self_attn.o_proj" ], "candidates": [ { "dkld": 1.8147029913961974e-05, "dbits": 125829120 } ] }, { "idx": 90, "layers": [ "model.layers.15.mlp.shared_experts.gate_proj", "model.layers.15.mlp.shared_experts.up_proj", "model.layers.15.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.577877840958492e-06, "dbits": 47185920 } ] }, { "idx": 91, "layers": [ "model.layers.15.mlp.experts.0.gate_proj", "model.layers.15.mlp.experts.1.gate_proj", "model.layers.15.mlp.experts.2.gate_proj", "model.layers.15.mlp.experts.3.gate_proj", "model.layers.15.mlp.experts.4.gate_proj", "model.layers.15.mlp.experts.5.gate_proj", "model.layers.15.mlp.experts.6.gate_proj", "model.layers.15.mlp.experts.7.gate_proj", "model.layers.15.mlp.experts.8.gate_proj", "model.layers.15.mlp.experts.9.gate_proj", "model.layers.15.mlp.experts.10.gate_proj", "model.layers.15.mlp.experts.11.gate_proj", "model.layers.15.mlp.experts.12.gate_proj", "model.layers.15.mlp.experts.13.gate_proj", "model.layers.15.mlp.experts.14.gate_proj", "model.layers.15.mlp.experts.15.gate_proj", "model.layers.15.mlp.experts.16.gate_proj", "model.layers.15.mlp.experts.17.gate_proj", "model.layers.15.mlp.experts.18.gate_proj", "model.layers.15.mlp.experts.19.gate_proj", "model.layers.15.mlp.experts.20.gate_proj", "model.layers.15.mlp.experts.21.gate_proj", "model.layers.15.mlp.experts.22.gate_proj", "model.layers.15.mlp.experts.23.gate_proj", "model.layers.15.mlp.experts.24.gate_proj", "model.layers.15.mlp.experts.25.gate_proj", "model.layers.15.mlp.experts.26.gate_proj", "model.layers.15.mlp.experts.27.gate_proj", "model.layers.15.mlp.experts.28.gate_proj", "model.layers.15.mlp.experts.29.gate_proj", "model.layers.15.mlp.experts.30.gate_proj", "model.layers.15.mlp.experts.31.gate_proj", "model.layers.15.mlp.experts.32.gate_proj", "model.layers.15.mlp.experts.33.gate_proj", "model.layers.15.mlp.experts.34.gate_proj", "model.layers.15.mlp.experts.35.gate_proj", "model.layers.15.mlp.experts.36.gate_proj", "model.layers.15.mlp.experts.37.gate_proj", "model.layers.15.mlp.experts.38.gate_proj", "model.layers.15.mlp.experts.39.gate_proj", "model.layers.15.mlp.experts.40.gate_proj", "model.layers.15.mlp.experts.41.gate_proj", "model.layers.15.mlp.experts.42.gate_proj", "model.layers.15.mlp.experts.43.gate_proj", "model.layers.15.mlp.experts.44.gate_proj", "model.layers.15.mlp.experts.45.gate_proj", "model.layers.15.mlp.experts.46.gate_proj", "model.layers.15.mlp.experts.47.gate_proj", "model.layers.15.mlp.experts.48.gate_proj", "model.layers.15.mlp.experts.49.gate_proj", "model.layers.15.mlp.experts.50.gate_proj", "model.layers.15.mlp.experts.51.gate_proj", "model.layers.15.mlp.experts.52.gate_proj", "model.layers.15.mlp.experts.53.gate_proj", "model.layers.15.mlp.experts.54.gate_proj", "model.layers.15.mlp.experts.55.gate_proj", "model.layers.15.mlp.experts.56.gate_proj", "model.layers.15.mlp.experts.57.gate_proj", "model.layers.15.mlp.experts.58.gate_proj", "model.layers.15.mlp.experts.59.gate_proj", "model.layers.15.mlp.experts.60.gate_proj", "model.layers.15.mlp.experts.61.gate_proj", "model.layers.15.mlp.experts.62.gate_proj", "model.layers.15.mlp.experts.63.gate_proj", "model.layers.15.mlp.experts.64.gate_proj", "model.layers.15.mlp.experts.65.gate_proj", "model.layers.15.mlp.experts.66.gate_proj", "model.layers.15.mlp.experts.67.gate_proj", "model.layers.15.mlp.experts.68.gate_proj", "model.layers.15.mlp.experts.69.gate_proj", "model.layers.15.mlp.experts.70.gate_proj", "model.layers.15.mlp.experts.71.gate_proj", "model.layers.15.mlp.experts.72.gate_proj", "model.layers.15.mlp.experts.73.gate_proj", "model.layers.15.mlp.experts.74.gate_proj", "model.layers.15.mlp.experts.75.gate_proj", "model.layers.15.mlp.experts.76.gate_proj", "model.layers.15.mlp.experts.77.gate_proj", "model.layers.15.mlp.experts.78.gate_proj", "model.layers.15.mlp.experts.79.gate_proj", "model.layers.15.mlp.experts.80.gate_proj", "model.layers.15.mlp.experts.81.gate_proj", "model.layers.15.mlp.experts.82.gate_proj", "model.layers.15.mlp.experts.83.gate_proj", "model.layers.15.mlp.experts.84.gate_proj", "model.layers.15.mlp.experts.85.gate_proj", "model.layers.15.mlp.experts.86.gate_proj", "model.layers.15.mlp.experts.87.gate_proj", "model.layers.15.mlp.experts.88.gate_proj", "model.layers.15.mlp.experts.89.gate_proj", "model.layers.15.mlp.experts.90.gate_proj", "model.layers.15.mlp.experts.91.gate_proj", "model.layers.15.mlp.experts.92.gate_proj", "model.layers.15.mlp.experts.93.gate_proj", "model.layers.15.mlp.experts.94.gate_proj", "model.layers.15.mlp.experts.95.gate_proj", "model.layers.15.mlp.experts.96.gate_proj", "model.layers.15.mlp.experts.97.gate_proj", "model.layers.15.mlp.experts.98.gate_proj", "model.layers.15.mlp.experts.99.gate_proj", "model.layers.15.mlp.experts.100.gate_proj", "model.layers.15.mlp.experts.101.gate_proj", "model.layers.15.mlp.experts.102.gate_proj", "model.layers.15.mlp.experts.103.gate_proj", "model.layers.15.mlp.experts.104.gate_proj", "model.layers.15.mlp.experts.105.gate_proj", "model.layers.15.mlp.experts.106.gate_proj", "model.layers.15.mlp.experts.107.gate_proj", "model.layers.15.mlp.experts.108.gate_proj", "model.layers.15.mlp.experts.109.gate_proj", "model.layers.15.mlp.experts.110.gate_proj", "model.layers.15.mlp.experts.111.gate_proj", "model.layers.15.mlp.experts.112.gate_proj", "model.layers.15.mlp.experts.113.gate_proj", "model.layers.15.mlp.experts.114.gate_proj", "model.layers.15.mlp.experts.115.gate_proj", "model.layers.15.mlp.experts.116.gate_proj", "model.layers.15.mlp.experts.117.gate_proj", "model.layers.15.mlp.experts.118.gate_proj", "model.layers.15.mlp.experts.119.gate_proj", "model.layers.15.mlp.experts.120.gate_proj", "model.layers.15.mlp.experts.121.gate_proj", "model.layers.15.mlp.experts.122.gate_proj", "model.layers.15.mlp.experts.123.gate_proj", "model.layers.15.mlp.experts.124.gate_proj", "model.layers.15.mlp.experts.125.gate_proj", "model.layers.15.mlp.experts.126.gate_proj", "model.layers.15.mlp.experts.127.gate_proj", "model.layers.15.mlp.experts.128.gate_proj", "model.layers.15.mlp.experts.129.gate_proj", "model.layers.15.mlp.experts.130.gate_proj", "model.layers.15.mlp.experts.131.gate_proj", "model.layers.15.mlp.experts.132.gate_proj", "model.layers.15.mlp.experts.133.gate_proj", "model.layers.15.mlp.experts.134.gate_proj", "model.layers.15.mlp.experts.135.gate_proj", "model.layers.15.mlp.experts.136.gate_proj", "model.layers.15.mlp.experts.137.gate_proj", "model.layers.15.mlp.experts.138.gate_proj", "model.layers.15.mlp.experts.139.gate_proj", "model.layers.15.mlp.experts.140.gate_proj", "model.layers.15.mlp.experts.141.gate_proj", "model.layers.15.mlp.experts.142.gate_proj", "model.layers.15.mlp.experts.143.gate_proj", "model.layers.15.mlp.experts.144.gate_proj", "model.layers.15.mlp.experts.145.gate_proj", "model.layers.15.mlp.experts.146.gate_proj", "model.layers.15.mlp.experts.147.gate_proj", "model.layers.15.mlp.experts.148.gate_proj", "model.layers.15.mlp.experts.149.gate_proj", "model.layers.15.mlp.experts.150.gate_proj", "model.layers.15.mlp.experts.151.gate_proj", "model.layers.15.mlp.experts.152.gate_proj", "model.layers.15.mlp.experts.153.gate_proj", "model.layers.15.mlp.experts.154.gate_proj", "model.layers.15.mlp.experts.155.gate_proj", "model.layers.15.mlp.experts.156.gate_proj", "model.layers.15.mlp.experts.157.gate_proj", "model.layers.15.mlp.experts.158.gate_proj", "model.layers.15.mlp.experts.159.gate_proj", "model.layers.15.mlp.experts.0.up_proj", "model.layers.15.mlp.experts.1.up_proj", "model.layers.15.mlp.experts.2.up_proj", "model.layers.15.mlp.experts.3.up_proj", "model.layers.15.mlp.experts.4.up_proj", "model.layers.15.mlp.experts.5.up_proj", "model.layers.15.mlp.experts.6.up_proj", "model.layers.15.mlp.experts.7.up_proj", "model.layers.15.mlp.experts.8.up_proj", "model.layers.15.mlp.experts.9.up_proj", "model.layers.15.mlp.experts.10.up_proj", "model.layers.15.mlp.experts.11.up_proj", "model.layers.15.mlp.experts.12.up_proj", "model.layers.15.mlp.experts.13.up_proj", "model.layers.15.mlp.experts.14.up_proj", "model.layers.15.mlp.experts.15.up_proj", "model.layers.15.mlp.experts.16.up_proj", "model.layers.15.mlp.experts.17.up_proj", "model.layers.15.mlp.experts.18.up_proj", "model.layers.15.mlp.experts.19.up_proj", "model.layers.15.mlp.experts.20.up_proj", "model.layers.15.mlp.experts.21.up_proj", "model.layers.15.mlp.experts.22.up_proj", "model.layers.15.mlp.experts.23.up_proj", "model.layers.15.mlp.experts.24.up_proj", "model.layers.15.mlp.experts.25.up_proj", "model.layers.15.mlp.experts.26.up_proj", "model.layers.15.mlp.experts.27.up_proj", "model.layers.15.mlp.experts.28.up_proj", "model.layers.15.mlp.experts.29.up_proj", "model.layers.15.mlp.experts.30.up_proj", "model.layers.15.mlp.experts.31.up_proj", "model.layers.15.mlp.experts.32.up_proj", "model.layers.15.mlp.experts.33.up_proj", "model.layers.15.mlp.experts.34.up_proj", "model.layers.15.mlp.experts.35.up_proj", "model.layers.15.mlp.experts.36.up_proj", "model.layers.15.mlp.experts.37.up_proj", "model.layers.15.mlp.experts.38.up_proj", "model.layers.15.mlp.experts.39.up_proj", "model.layers.15.mlp.experts.40.up_proj", "model.layers.15.mlp.experts.41.up_proj", "model.layers.15.mlp.experts.42.up_proj", "model.layers.15.mlp.experts.43.up_proj", "model.layers.15.mlp.experts.44.up_proj", "model.layers.15.mlp.experts.45.up_proj", "model.layers.15.mlp.experts.46.up_proj", "model.layers.15.mlp.experts.47.up_proj", "model.layers.15.mlp.experts.48.up_proj", "model.layers.15.mlp.experts.49.up_proj", "model.layers.15.mlp.experts.50.up_proj", "model.layers.15.mlp.experts.51.up_proj", "model.layers.15.mlp.experts.52.up_proj", "model.layers.15.mlp.experts.53.up_proj", "model.layers.15.mlp.experts.54.up_proj", "model.layers.15.mlp.experts.55.up_proj", "model.layers.15.mlp.experts.56.up_proj", "model.layers.15.mlp.experts.57.up_proj", "model.layers.15.mlp.experts.58.up_proj", "model.layers.15.mlp.experts.59.up_proj", "model.layers.15.mlp.experts.60.up_proj", "model.layers.15.mlp.experts.61.up_proj", "model.layers.15.mlp.experts.62.up_proj", "model.layers.15.mlp.experts.63.up_proj", "model.layers.15.mlp.experts.64.up_proj", "model.layers.15.mlp.experts.65.up_proj", "model.layers.15.mlp.experts.66.up_proj", "model.layers.15.mlp.experts.67.up_proj", "model.layers.15.mlp.experts.68.up_proj", "model.layers.15.mlp.experts.69.up_proj", "model.layers.15.mlp.experts.70.up_proj", "model.layers.15.mlp.experts.71.up_proj", "model.layers.15.mlp.experts.72.up_proj", "model.layers.15.mlp.experts.73.up_proj", "model.layers.15.mlp.experts.74.up_proj", "model.layers.15.mlp.experts.75.up_proj", "model.layers.15.mlp.experts.76.up_proj", "model.layers.15.mlp.experts.77.up_proj", "model.layers.15.mlp.experts.78.up_proj", "model.layers.15.mlp.experts.79.up_proj", "model.layers.15.mlp.experts.80.up_proj", "model.layers.15.mlp.experts.81.up_proj", "model.layers.15.mlp.experts.82.up_proj", "model.layers.15.mlp.experts.83.up_proj", "model.layers.15.mlp.experts.84.up_proj", "model.layers.15.mlp.experts.85.up_proj", "model.layers.15.mlp.experts.86.up_proj", "model.layers.15.mlp.experts.87.up_proj", "model.layers.15.mlp.experts.88.up_proj", "model.layers.15.mlp.experts.89.up_proj", "model.layers.15.mlp.experts.90.up_proj", "model.layers.15.mlp.experts.91.up_proj", "model.layers.15.mlp.experts.92.up_proj", "model.layers.15.mlp.experts.93.up_proj", "model.layers.15.mlp.experts.94.up_proj", "model.layers.15.mlp.experts.95.up_proj", "model.layers.15.mlp.experts.96.up_proj", "model.layers.15.mlp.experts.97.up_proj", "model.layers.15.mlp.experts.98.up_proj", "model.layers.15.mlp.experts.99.up_proj", "model.layers.15.mlp.experts.100.up_proj", "model.layers.15.mlp.experts.101.up_proj", "model.layers.15.mlp.experts.102.up_proj", "model.layers.15.mlp.experts.103.up_proj", "model.layers.15.mlp.experts.104.up_proj", "model.layers.15.mlp.experts.105.up_proj", "model.layers.15.mlp.experts.106.up_proj", "model.layers.15.mlp.experts.107.up_proj", "model.layers.15.mlp.experts.108.up_proj", "model.layers.15.mlp.experts.109.up_proj", "model.layers.15.mlp.experts.110.up_proj", "model.layers.15.mlp.experts.111.up_proj", "model.layers.15.mlp.experts.112.up_proj", "model.layers.15.mlp.experts.113.up_proj", "model.layers.15.mlp.experts.114.up_proj", "model.layers.15.mlp.experts.115.up_proj", "model.layers.15.mlp.experts.116.up_proj", "model.layers.15.mlp.experts.117.up_proj", "model.layers.15.mlp.experts.118.up_proj", "model.layers.15.mlp.experts.119.up_proj", "model.layers.15.mlp.experts.120.up_proj", "model.layers.15.mlp.experts.121.up_proj", "model.layers.15.mlp.experts.122.up_proj", "model.layers.15.mlp.experts.123.up_proj", "model.layers.15.mlp.experts.124.up_proj", "model.layers.15.mlp.experts.125.up_proj", "model.layers.15.mlp.experts.126.up_proj", "model.layers.15.mlp.experts.127.up_proj", "model.layers.15.mlp.experts.128.up_proj", "model.layers.15.mlp.experts.129.up_proj", "model.layers.15.mlp.experts.130.up_proj", "model.layers.15.mlp.experts.131.up_proj", "model.layers.15.mlp.experts.132.up_proj", "model.layers.15.mlp.experts.133.up_proj", "model.layers.15.mlp.experts.134.up_proj", "model.layers.15.mlp.experts.135.up_proj", "model.layers.15.mlp.experts.136.up_proj", "model.layers.15.mlp.experts.137.up_proj", "model.layers.15.mlp.experts.138.up_proj", "model.layers.15.mlp.experts.139.up_proj", "model.layers.15.mlp.experts.140.up_proj", "model.layers.15.mlp.experts.141.up_proj", "model.layers.15.mlp.experts.142.up_proj", "model.layers.15.mlp.experts.143.up_proj", "model.layers.15.mlp.experts.144.up_proj", "model.layers.15.mlp.experts.145.up_proj", "model.layers.15.mlp.experts.146.up_proj", "model.layers.15.mlp.experts.147.up_proj", "model.layers.15.mlp.experts.148.up_proj", "model.layers.15.mlp.experts.149.up_proj", "model.layers.15.mlp.experts.150.up_proj", "model.layers.15.mlp.experts.151.up_proj", "model.layers.15.mlp.experts.152.up_proj", "model.layers.15.mlp.experts.153.up_proj", "model.layers.15.mlp.experts.154.up_proj", "model.layers.15.mlp.experts.155.up_proj", "model.layers.15.mlp.experts.156.up_proj", "model.layers.15.mlp.experts.157.up_proj", "model.layers.15.mlp.experts.158.up_proj", "model.layers.15.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.6948564229533293e-06, "dbits": 5033164800 } ] }, { "idx": 92, "layers": [ "model.layers.15.mlp.experts.0.down_proj", "model.layers.15.mlp.experts.1.down_proj", "model.layers.15.mlp.experts.2.down_proj", "model.layers.15.mlp.experts.3.down_proj", "model.layers.15.mlp.experts.4.down_proj", "model.layers.15.mlp.experts.5.down_proj", "model.layers.15.mlp.experts.6.down_proj", "model.layers.15.mlp.experts.7.down_proj", "model.layers.15.mlp.experts.8.down_proj", "model.layers.15.mlp.experts.9.down_proj", "model.layers.15.mlp.experts.10.down_proj", "model.layers.15.mlp.experts.11.down_proj", "model.layers.15.mlp.experts.12.down_proj", "model.layers.15.mlp.experts.13.down_proj", "model.layers.15.mlp.experts.14.down_proj", "model.layers.15.mlp.experts.15.down_proj", "model.layers.15.mlp.experts.16.down_proj", "model.layers.15.mlp.experts.17.down_proj", "model.layers.15.mlp.experts.18.down_proj", "model.layers.15.mlp.experts.19.down_proj", "model.layers.15.mlp.experts.20.down_proj", "model.layers.15.mlp.experts.21.down_proj", "model.layers.15.mlp.experts.22.down_proj", "model.layers.15.mlp.experts.23.down_proj", "model.layers.15.mlp.experts.24.down_proj", "model.layers.15.mlp.experts.25.down_proj", "model.layers.15.mlp.experts.26.down_proj", "model.layers.15.mlp.experts.27.down_proj", "model.layers.15.mlp.experts.28.down_proj", "model.layers.15.mlp.experts.29.down_proj", "model.layers.15.mlp.experts.30.down_proj", "model.layers.15.mlp.experts.31.down_proj", "model.layers.15.mlp.experts.32.down_proj", "model.layers.15.mlp.experts.33.down_proj", "model.layers.15.mlp.experts.34.down_proj", "model.layers.15.mlp.experts.35.down_proj", "model.layers.15.mlp.experts.36.down_proj", "model.layers.15.mlp.experts.37.down_proj", "model.layers.15.mlp.experts.38.down_proj", "model.layers.15.mlp.experts.39.down_proj", "model.layers.15.mlp.experts.40.down_proj", "model.layers.15.mlp.experts.41.down_proj", "model.layers.15.mlp.experts.42.down_proj", "model.layers.15.mlp.experts.43.down_proj", "model.layers.15.mlp.experts.44.down_proj", "model.layers.15.mlp.experts.45.down_proj", "model.layers.15.mlp.experts.46.down_proj", "model.layers.15.mlp.experts.47.down_proj", "model.layers.15.mlp.experts.48.down_proj", "model.layers.15.mlp.experts.49.down_proj", "model.layers.15.mlp.experts.50.down_proj", "model.layers.15.mlp.experts.51.down_proj", "model.layers.15.mlp.experts.52.down_proj", "model.layers.15.mlp.experts.53.down_proj", "model.layers.15.mlp.experts.54.down_proj", "model.layers.15.mlp.experts.55.down_proj", "model.layers.15.mlp.experts.56.down_proj", "model.layers.15.mlp.experts.57.down_proj", "model.layers.15.mlp.experts.58.down_proj", "model.layers.15.mlp.experts.59.down_proj", "model.layers.15.mlp.experts.60.down_proj", "model.layers.15.mlp.experts.61.down_proj", "model.layers.15.mlp.experts.62.down_proj", "model.layers.15.mlp.experts.63.down_proj", "model.layers.15.mlp.experts.64.down_proj", "model.layers.15.mlp.experts.65.down_proj", "model.layers.15.mlp.experts.66.down_proj", "model.layers.15.mlp.experts.67.down_proj", "model.layers.15.mlp.experts.68.down_proj", "model.layers.15.mlp.experts.69.down_proj", "model.layers.15.mlp.experts.70.down_proj", "model.layers.15.mlp.experts.71.down_proj", "model.layers.15.mlp.experts.72.down_proj", "model.layers.15.mlp.experts.73.down_proj", "model.layers.15.mlp.experts.74.down_proj", "model.layers.15.mlp.experts.75.down_proj", "model.layers.15.mlp.experts.76.down_proj", "model.layers.15.mlp.experts.77.down_proj", "model.layers.15.mlp.experts.78.down_proj", "model.layers.15.mlp.experts.79.down_proj", "model.layers.15.mlp.experts.80.down_proj", "model.layers.15.mlp.experts.81.down_proj", "model.layers.15.mlp.experts.82.down_proj", "model.layers.15.mlp.experts.83.down_proj", "model.layers.15.mlp.experts.84.down_proj", "model.layers.15.mlp.experts.85.down_proj", "model.layers.15.mlp.experts.86.down_proj", "model.layers.15.mlp.experts.87.down_proj", "model.layers.15.mlp.experts.88.down_proj", "model.layers.15.mlp.experts.89.down_proj", "model.layers.15.mlp.experts.90.down_proj", "model.layers.15.mlp.experts.91.down_proj", "model.layers.15.mlp.experts.92.down_proj", "model.layers.15.mlp.experts.93.down_proj", "model.layers.15.mlp.experts.94.down_proj", "model.layers.15.mlp.experts.95.down_proj", "model.layers.15.mlp.experts.96.down_proj", "model.layers.15.mlp.experts.97.down_proj", "model.layers.15.mlp.experts.98.down_proj", "model.layers.15.mlp.experts.99.down_proj", "model.layers.15.mlp.experts.100.down_proj", "model.layers.15.mlp.experts.101.down_proj", "model.layers.15.mlp.experts.102.down_proj", "model.layers.15.mlp.experts.103.down_proj", "model.layers.15.mlp.experts.104.down_proj", "model.layers.15.mlp.experts.105.down_proj", "model.layers.15.mlp.experts.106.down_proj", "model.layers.15.mlp.experts.107.down_proj", "model.layers.15.mlp.experts.108.down_proj", "model.layers.15.mlp.experts.109.down_proj", "model.layers.15.mlp.experts.110.down_proj", "model.layers.15.mlp.experts.111.down_proj", "model.layers.15.mlp.experts.112.down_proj", "model.layers.15.mlp.experts.113.down_proj", "model.layers.15.mlp.experts.114.down_proj", "model.layers.15.mlp.experts.115.down_proj", "model.layers.15.mlp.experts.116.down_proj", "model.layers.15.mlp.experts.117.down_proj", "model.layers.15.mlp.experts.118.down_proj", "model.layers.15.mlp.experts.119.down_proj", "model.layers.15.mlp.experts.120.down_proj", "model.layers.15.mlp.experts.121.down_proj", "model.layers.15.mlp.experts.122.down_proj", "model.layers.15.mlp.experts.123.down_proj", "model.layers.15.mlp.experts.124.down_proj", "model.layers.15.mlp.experts.125.down_proj", "model.layers.15.mlp.experts.126.down_proj", "model.layers.15.mlp.experts.127.down_proj", "model.layers.15.mlp.experts.128.down_proj", "model.layers.15.mlp.experts.129.down_proj", "model.layers.15.mlp.experts.130.down_proj", "model.layers.15.mlp.experts.131.down_proj", "model.layers.15.mlp.experts.132.down_proj", "model.layers.15.mlp.experts.133.down_proj", "model.layers.15.mlp.experts.134.down_proj", "model.layers.15.mlp.experts.135.down_proj", "model.layers.15.mlp.experts.136.down_proj", "model.layers.15.mlp.experts.137.down_proj", "model.layers.15.mlp.experts.138.down_proj", "model.layers.15.mlp.experts.139.down_proj", "model.layers.15.mlp.experts.140.down_proj", "model.layers.15.mlp.experts.141.down_proj", "model.layers.15.mlp.experts.142.down_proj", "model.layers.15.mlp.experts.143.down_proj", "model.layers.15.mlp.experts.144.down_proj", "model.layers.15.mlp.experts.145.down_proj", "model.layers.15.mlp.experts.146.down_proj", "model.layers.15.mlp.experts.147.down_proj", "model.layers.15.mlp.experts.148.down_proj", "model.layers.15.mlp.experts.149.down_proj", "model.layers.15.mlp.experts.150.down_proj", "model.layers.15.mlp.experts.151.down_proj", "model.layers.15.mlp.experts.152.down_proj", "model.layers.15.mlp.experts.153.down_proj", "model.layers.15.mlp.experts.154.down_proj", "model.layers.15.mlp.experts.155.down_proj", "model.layers.15.mlp.experts.156.down_proj", "model.layers.15.mlp.experts.157.down_proj", "model.layers.15.mlp.experts.158.down_proj", "model.layers.15.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.3201602268964096e-05, "dbits": 2516582400 } ] }, { "idx": 93, "layers": [ "model.layers.16.self_attn.q_proj" ], "candidates": [ { "dkld": 7.925080717541364e-06, "dbits": 125829120 } ] }, { "idx": 94, "layers": [ "model.layers.16.self_attn.k_proj", "model.layers.16.self_attn.v_proj" ], "candidates": [ { "dkld": 9.718173532746689e-06, "dbits": 20971520 } ] }, { "idx": 95, "layers": [ "model.layers.16.self_attn.o_proj" ], "candidates": [ { "dkld": 1.5950811211951164e-05, "dbits": 125829120 } ] }, { "idx": 96, "layers": [ "model.layers.16.mlp.shared_experts.gate_proj", "model.layers.16.mlp.shared_experts.up_proj", "model.layers.16.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.6661491827107966e-05, "dbits": 47185920 } ] }, { "idx": 97, "layers": [ "model.layers.16.mlp.experts.0.gate_proj", "model.layers.16.mlp.experts.1.gate_proj", "model.layers.16.mlp.experts.2.gate_proj", "model.layers.16.mlp.experts.3.gate_proj", "model.layers.16.mlp.experts.4.gate_proj", "model.layers.16.mlp.experts.5.gate_proj", "model.layers.16.mlp.experts.6.gate_proj", "model.layers.16.mlp.experts.7.gate_proj", "model.layers.16.mlp.experts.8.gate_proj", "model.layers.16.mlp.experts.9.gate_proj", "model.layers.16.mlp.experts.10.gate_proj", "model.layers.16.mlp.experts.11.gate_proj", "model.layers.16.mlp.experts.12.gate_proj", "model.layers.16.mlp.experts.13.gate_proj", "model.layers.16.mlp.experts.14.gate_proj", "model.layers.16.mlp.experts.15.gate_proj", "model.layers.16.mlp.experts.16.gate_proj", "model.layers.16.mlp.experts.17.gate_proj", "model.layers.16.mlp.experts.18.gate_proj", "model.layers.16.mlp.experts.19.gate_proj", "model.layers.16.mlp.experts.20.gate_proj", "model.layers.16.mlp.experts.21.gate_proj", "model.layers.16.mlp.experts.22.gate_proj", "model.layers.16.mlp.experts.23.gate_proj", "model.layers.16.mlp.experts.24.gate_proj", "model.layers.16.mlp.experts.25.gate_proj", "model.layers.16.mlp.experts.26.gate_proj", "model.layers.16.mlp.experts.27.gate_proj", "model.layers.16.mlp.experts.28.gate_proj", "model.layers.16.mlp.experts.29.gate_proj", "model.layers.16.mlp.experts.30.gate_proj", "model.layers.16.mlp.experts.31.gate_proj", "model.layers.16.mlp.experts.32.gate_proj", "model.layers.16.mlp.experts.33.gate_proj", "model.layers.16.mlp.experts.34.gate_proj", "model.layers.16.mlp.experts.35.gate_proj", "model.layers.16.mlp.experts.36.gate_proj", "model.layers.16.mlp.experts.37.gate_proj", "model.layers.16.mlp.experts.38.gate_proj", "model.layers.16.mlp.experts.39.gate_proj", "model.layers.16.mlp.experts.40.gate_proj", "model.layers.16.mlp.experts.41.gate_proj", "model.layers.16.mlp.experts.42.gate_proj", "model.layers.16.mlp.experts.43.gate_proj", "model.layers.16.mlp.experts.44.gate_proj", "model.layers.16.mlp.experts.45.gate_proj", "model.layers.16.mlp.experts.46.gate_proj", "model.layers.16.mlp.experts.47.gate_proj", "model.layers.16.mlp.experts.48.gate_proj", "model.layers.16.mlp.experts.49.gate_proj", "model.layers.16.mlp.experts.50.gate_proj", "model.layers.16.mlp.experts.51.gate_proj", "model.layers.16.mlp.experts.52.gate_proj", "model.layers.16.mlp.experts.53.gate_proj", "model.layers.16.mlp.experts.54.gate_proj", "model.layers.16.mlp.experts.55.gate_proj", "model.layers.16.mlp.experts.56.gate_proj", "model.layers.16.mlp.experts.57.gate_proj", "model.layers.16.mlp.experts.58.gate_proj", "model.layers.16.mlp.experts.59.gate_proj", "model.layers.16.mlp.experts.60.gate_proj", "model.layers.16.mlp.experts.61.gate_proj", "model.layers.16.mlp.experts.62.gate_proj", "model.layers.16.mlp.experts.63.gate_proj", "model.layers.16.mlp.experts.64.gate_proj", "model.layers.16.mlp.experts.65.gate_proj", "model.layers.16.mlp.experts.66.gate_proj", "model.layers.16.mlp.experts.67.gate_proj", "model.layers.16.mlp.experts.68.gate_proj", "model.layers.16.mlp.experts.69.gate_proj", "model.layers.16.mlp.experts.70.gate_proj", "model.layers.16.mlp.experts.71.gate_proj", "model.layers.16.mlp.experts.72.gate_proj", "model.layers.16.mlp.experts.73.gate_proj", "model.layers.16.mlp.experts.74.gate_proj", "model.layers.16.mlp.experts.75.gate_proj", "model.layers.16.mlp.experts.76.gate_proj", "model.layers.16.mlp.experts.77.gate_proj", "model.layers.16.mlp.experts.78.gate_proj", "model.layers.16.mlp.experts.79.gate_proj", "model.layers.16.mlp.experts.80.gate_proj", "model.layers.16.mlp.experts.81.gate_proj", "model.layers.16.mlp.experts.82.gate_proj", "model.layers.16.mlp.experts.83.gate_proj", "model.layers.16.mlp.experts.84.gate_proj", "model.layers.16.mlp.experts.85.gate_proj", "model.layers.16.mlp.experts.86.gate_proj", "model.layers.16.mlp.experts.87.gate_proj", "model.layers.16.mlp.experts.88.gate_proj", "model.layers.16.mlp.experts.89.gate_proj", "model.layers.16.mlp.experts.90.gate_proj", "model.layers.16.mlp.experts.91.gate_proj", "model.layers.16.mlp.experts.92.gate_proj", "model.layers.16.mlp.experts.93.gate_proj", "model.layers.16.mlp.experts.94.gate_proj", "model.layers.16.mlp.experts.95.gate_proj", "model.layers.16.mlp.experts.96.gate_proj", "model.layers.16.mlp.experts.97.gate_proj", "model.layers.16.mlp.experts.98.gate_proj", "model.layers.16.mlp.experts.99.gate_proj", "model.layers.16.mlp.experts.100.gate_proj", "model.layers.16.mlp.experts.101.gate_proj", "model.layers.16.mlp.experts.102.gate_proj", "model.layers.16.mlp.experts.103.gate_proj", "model.layers.16.mlp.experts.104.gate_proj", "model.layers.16.mlp.experts.105.gate_proj", "model.layers.16.mlp.experts.106.gate_proj", "model.layers.16.mlp.experts.107.gate_proj", "model.layers.16.mlp.experts.108.gate_proj", "model.layers.16.mlp.experts.109.gate_proj", "model.layers.16.mlp.experts.110.gate_proj", "model.layers.16.mlp.experts.111.gate_proj", "model.layers.16.mlp.experts.112.gate_proj", "model.layers.16.mlp.experts.113.gate_proj", "model.layers.16.mlp.experts.114.gate_proj", "model.layers.16.mlp.experts.115.gate_proj", "model.layers.16.mlp.experts.116.gate_proj", "model.layers.16.mlp.experts.117.gate_proj", "model.layers.16.mlp.experts.118.gate_proj", "model.layers.16.mlp.experts.119.gate_proj", "model.layers.16.mlp.experts.120.gate_proj", "model.layers.16.mlp.experts.121.gate_proj", "model.layers.16.mlp.experts.122.gate_proj", "model.layers.16.mlp.experts.123.gate_proj", "model.layers.16.mlp.experts.124.gate_proj", "model.layers.16.mlp.experts.125.gate_proj", "model.layers.16.mlp.experts.126.gate_proj", "model.layers.16.mlp.experts.127.gate_proj", "model.layers.16.mlp.experts.128.gate_proj", "model.layers.16.mlp.experts.129.gate_proj", "model.layers.16.mlp.experts.130.gate_proj", "model.layers.16.mlp.experts.131.gate_proj", "model.layers.16.mlp.experts.132.gate_proj", "model.layers.16.mlp.experts.133.gate_proj", "model.layers.16.mlp.experts.134.gate_proj", "model.layers.16.mlp.experts.135.gate_proj", "model.layers.16.mlp.experts.136.gate_proj", "model.layers.16.mlp.experts.137.gate_proj", "model.layers.16.mlp.experts.138.gate_proj", "model.layers.16.mlp.experts.139.gate_proj", "model.layers.16.mlp.experts.140.gate_proj", "model.layers.16.mlp.experts.141.gate_proj", "model.layers.16.mlp.experts.142.gate_proj", "model.layers.16.mlp.experts.143.gate_proj", "model.layers.16.mlp.experts.144.gate_proj", "model.layers.16.mlp.experts.145.gate_proj", "model.layers.16.mlp.experts.146.gate_proj", "model.layers.16.mlp.experts.147.gate_proj", "model.layers.16.mlp.experts.148.gate_proj", "model.layers.16.mlp.experts.149.gate_proj", "model.layers.16.mlp.experts.150.gate_proj", "model.layers.16.mlp.experts.151.gate_proj", "model.layers.16.mlp.experts.152.gate_proj", "model.layers.16.mlp.experts.153.gate_proj", "model.layers.16.mlp.experts.154.gate_proj", "model.layers.16.mlp.experts.155.gate_proj", "model.layers.16.mlp.experts.156.gate_proj", "model.layers.16.mlp.experts.157.gate_proj", "model.layers.16.mlp.experts.158.gate_proj", "model.layers.16.mlp.experts.159.gate_proj", "model.layers.16.mlp.experts.0.up_proj", "model.layers.16.mlp.experts.1.up_proj", "model.layers.16.mlp.experts.2.up_proj", "model.layers.16.mlp.experts.3.up_proj", "model.layers.16.mlp.experts.4.up_proj", "model.layers.16.mlp.experts.5.up_proj", "model.layers.16.mlp.experts.6.up_proj", "model.layers.16.mlp.experts.7.up_proj", "model.layers.16.mlp.experts.8.up_proj", "model.layers.16.mlp.experts.9.up_proj", "model.layers.16.mlp.experts.10.up_proj", "model.layers.16.mlp.experts.11.up_proj", "model.layers.16.mlp.experts.12.up_proj", "model.layers.16.mlp.experts.13.up_proj", "model.layers.16.mlp.experts.14.up_proj", "model.layers.16.mlp.experts.15.up_proj", "model.layers.16.mlp.experts.16.up_proj", "model.layers.16.mlp.experts.17.up_proj", "model.layers.16.mlp.experts.18.up_proj", "model.layers.16.mlp.experts.19.up_proj", "model.layers.16.mlp.experts.20.up_proj", "model.layers.16.mlp.experts.21.up_proj", "model.layers.16.mlp.experts.22.up_proj", "model.layers.16.mlp.experts.23.up_proj", "model.layers.16.mlp.experts.24.up_proj", "model.layers.16.mlp.experts.25.up_proj", "model.layers.16.mlp.experts.26.up_proj", "model.layers.16.mlp.experts.27.up_proj", "model.layers.16.mlp.experts.28.up_proj", "model.layers.16.mlp.experts.29.up_proj", "model.layers.16.mlp.experts.30.up_proj", "model.layers.16.mlp.experts.31.up_proj", "model.layers.16.mlp.experts.32.up_proj", "model.layers.16.mlp.experts.33.up_proj", "model.layers.16.mlp.experts.34.up_proj", "model.layers.16.mlp.experts.35.up_proj", "model.layers.16.mlp.experts.36.up_proj", "model.layers.16.mlp.experts.37.up_proj", "model.layers.16.mlp.experts.38.up_proj", "model.layers.16.mlp.experts.39.up_proj", "model.layers.16.mlp.experts.40.up_proj", "model.layers.16.mlp.experts.41.up_proj", "model.layers.16.mlp.experts.42.up_proj", "model.layers.16.mlp.experts.43.up_proj", "model.layers.16.mlp.experts.44.up_proj", "model.layers.16.mlp.experts.45.up_proj", "model.layers.16.mlp.experts.46.up_proj", "model.layers.16.mlp.experts.47.up_proj", "model.layers.16.mlp.experts.48.up_proj", "model.layers.16.mlp.experts.49.up_proj", "model.layers.16.mlp.experts.50.up_proj", "model.layers.16.mlp.experts.51.up_proj", "model.layers.16.mlp.experts.52.up_proj", "model.layers.16.mlp.experts.53.up_proj", "model.layers.16.mlp.experts.54.up_proj", "model.layers.16.mlp.experts.55.up_proj", "model.layers.16.mlp.experts.56.up_proj", "model.layers.16.mlp.experts.57.up_proj", "model.layers.16.mlp.experts.58.up_proj", "model.layers.16.mlp.experts.59.up_proj", "model.layers.16.mlp.experts.60.up_proj", "model.layers.16.mlp.experts.61.up_proj", "model.layers.16.mlp.experts.62.up_proj", "model.layers.16.mlp.experts.63.up_proj", "model.layers.16.mlp.experts.64.up_proj", "model.layers.16.mlp.experts.65.up_proj", "model.layers.16.mlp.experts.66.up_proj", "model.layers.16.mlp.experts.67.up_proj", "model.layers.16.mlp.experts.68.up_proj", "model.layers.16.mlp.experts.69.up_proj", "model.layers.16.mlp.experts.70.up_proj", "model.layers.16.mlp.experts.71.up_proj", "model.layers.16.mlp.experts.72.up_proj", "model.layers.16.mlp.experts.73.up_proj", "model.layers.16.mlp.experts.74.up_proj", "model.layers.16.mlp.experts.75.up_proj", "model.layers.16.mlp.experts.76.up_proj", "model.layers.16.mlp.experts.77.up_proj", "model.layers.16.mlp.experts.78.up_proj", "model.layers.16.mlp.experts.79.up_proj", "model.layers.16.mlp.experts.80.up_proj", "model.layers.16.mlp.experts.81.up_proj", "model.layers.16.mlp.experts.82.up_proj", "model.layers.16.mlp.experts.83.up_proj", "model.layers.16.mlp.experts.84.up_proj", "model.layers.16.mlp.experts.85.up_proj", "model.layers.16.mlp.experts.86.up_proj", "model.layers.16.mlp.experts.87.up_proj", "model.layers.16.mlp.experts.88.up_proj", "model.layers.16.mlp.experts.89.up_proj", "model.layers.16.mlp.experts.90.up_proj", "model.layers.16.mlp.experts.91.up_proj", "model.layers.16.mlp.experts.92.up_proj", "model.layers.16.mlp.experts.93.up_proj", "model.layers.16.mlp.experts.94.up_proj", "model.layers.16.mlp.experts.95.up_proj", "model.layers.16.mlp.experts.96.up_proj", "model.layers.16.mlp.experts.97.up_proj", "model.layers.16.mlp.experts.98.up_proj", "model.layers.16.mlp.experts.99.up_proj", "model.layers.16.mlp.experts.100.up_proj", "model.layers.16.mlp.experts.101.up_proj", "model.layers.16.mlp.experts.102.up_proj", "model.layers.16.mlp.experts.103.up_proj", "model.layers.16.mlp.experts.104.up_proj", "model.layers.16.mlp.experts.105.up_proj", "model.layers.16.mlp.experts.106.up_proj", "model.layers.16.mlp.experts.107.up_proj", "model.layers.16.mlp.experts.108.up_proj", "model.layers.16.mlp.experts.109.up_proj", "model.layers.16.mlp.experts.110.up_proj", "model.layers.16.mlp.experts.111.up_proj", "model.layers.16.mlp.experts.112.up_proj", "model.layers.16.mlp.experts.113.up_proj", "model.layers.16.mlp.experts.114.up_proj", "model.layers.16.mlp.experts.115.up_proj", "model.layers.16.mlp.experts.116.up_proj", "model.layers.16.mlp.experts.117.up_proj", "model.layers.16.mlp.experts.118.up_proj", "model.layers.16.mlp.experts.119.up_proj", "model.layers.16.mlp.experts.120.up_proj", "model.layers.16.mlp.experts.121.up_proj", "model.layers.16.mlp.experts.122.up_proj", "model.layers.16.mlp.experts.123.up_proj", "model.layers.16.mlp.experts.124.up_proj", "model.layers.16.mlp.experts.125.up_proj", "model.layers.16.mlp.experts.126.up_proj", "model.layers.16.mlp.experts.127.up_proj", "model.layers.16.mlp.experts.128.up_proj", "model.layers.16.mlp.experts.129.up_proj", "model.layers.16.mlp.experts.130.up_proj", "model.layers.16.mlp.experts.131.up_proj", "model.layers.16.mlp.experts.132.up_proj", "model.layers.16.mlp.experts.133.up_proj", "model.layers.16.mlp.experts.134.up_proj", "model.layers.16.mlp.experts.135.up_proj", "model.layers.16.mlp.experts.136.up_proj", "model.layers.16.mlp.experts.137.up_proj", "model.layers.16.mlp.experts.138.up_proj", "model.layers.16.mlp.experts.139.up_proj", "model.layers.16.mlp.experts.140.up_proj", "model.layers.16.mlp.experts.141.up_proj", "model.layers.16.mlp.experts.142.up_proj", "model.layers.16.mlp.experts.143.up_proj", "model.layers.16.mlp.experts.144.up_proj", "model.layers.16.mlp.experts.145.up_proj", "model.layers.16.mlp.experts.146.up_proj", "model.layers.16.mlp.experts.147.up_proj", "model.layers.16.mlp.experts.148.up_proj", "model.layers.16.mlp.experts.149.up_proj", "model.layers.16.mlp.experts.150.up_proj", "model.layers.16.mlp.experts.151.up_proj", "model.layers.16.mlp.experts.152.up_proj", "model.layers.16.mlp.experts.153.up_proj", "model.layers.16.mlp.experts.154.up_proj", "model.layers.16.mlp.experts.155.up_proj", "model.layers.16.mlp.experts.156.up_proj", "model.layers.16.mlp.experts.157.up_proj", "model.layers.16.mlp.experts.158.up_proj", "model.layers.16.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.74583023181186e-05, "dbits": 5033164800 } ] }, { "idx": 98, "layers": [ "model.layers.16.mlp.experts.0.down_proj", "model.layers.16.mlp.experts.1.down_proj", "model.layers.16.mlp.experts.2.down_proj", "model.layers.16.mlp.experts.3.down_proj", "model.layers.16.mlp.experts.4.down_proj", "model.layers.16.mlp.experts.5.down_proj", "model.layers.16.mlp.experts.6.down_proj", "model.layers.16.mlp.experts.7.down_proj", "model.layers.16.mlp.experts.8.down_proj", "model.layers.16.mlp.experts.9.down_proj", "model.layers.16.mlp.experts.10.down_proj", "model.layers.16.mlp.experts.11.down_proj", "model.layers.16.mlp.experts.12.down_proj", "model.layers.16.mlp.experts.13.down_proj", "model.layers.16.mlp.experts.14.down_proj", "model.layers.16.mlp.experts.15.down_proj", "model.layers.16.mlp.experts.16.down_proj", "model.layers.16.mlp.experts.17.down_proj", "model.layers.16.mlp.experts.18.down_proj", "model.layers.16.mlp.experts.19.down_proj", "model.layers.16.mlp.experts.20.down_proj", "model.layers.16.mlp.experts.21.down_proj", "model.layers.16.mlp.experts.22.down_proj", "model.layers.16.mlp.experts.23.down_proj", "model.layers.16.mlp.experts.24.down_proj", "model.layers.16.mlp.experts.25.down_proj", "model.layers.16.mlp.experts.26.down_proj", "model.layers.16.mlp.experts.27.down_proj", "model.layers.16.mlp.experts.28.down_proj", "model.layers.16.mlp.experts.29.down_proj", "model.layers.16.mlp.experts.30.down_proj", "model.layers.16.mlp.experts.31.down_proj", "model.layers.16.mlp.experts.32.down_proj", "model.layers.16.mlp.experts.33.down_proj", "model.layers.16.mlp.experts.34.down_proj", "model.layers.16.mlp.experts.35.down_proj", "model.layers.16.mlp.experts.36.down_proj", "model.layers.16.mlp.experts.37.down_proj", "model.layers.16.mlp.experts.38.down_proj", "model.layers.16.mlp.experts.39.down_proj", "model.layers.16.mlp.experts.40.down_proj", "model.layers.16.mlp.experts.41.down_proj", "model.layers.16.mlp.experts.42.down_proj", "model.layers.16.mlp.experts.43.down_proj", "model.layers.16.mlp.experts.44.down_proj", "model.layers.16.mlp.experts.45.down_proj", "model.layers.16.mlp.experts.46.down_proj", "model.layers.16.mlp.experts.47.down_proj", "model.layers.16.mlp.experts.48.down_proj", "model.layers.16.mlp.experts.49.down_proj", "model.layers.16.mlp.experts.50.down_proj", "model.layers.16.mlp.experts.51.down_proj", "model.layers.16.mlp.experts.52.down_proj", "model.layers.16.mlp.experts.53.down_proj", "model.layers.16.mlp.experts.54.down_proj", "model.layers.16.mlp.experts.55.down_proj", "model.layers.16.mlp.experts.56.down_proj", "model.layers.16.mlp.experts.57.down_proj", "model.layers.16.mlp.experts.58.down_proj", "model.layers.16.mlp.experts.59.down_proj", "model.layers.16.mlp.experts.60.down_proj", "model.layers.16.mlp.experts.61.down_proj", "model.layers.16.mlp.experts.62.down_proj", "model.layers.16.mlp.experts.63.down_proj", "model.layers.16.mlp.experts.64.down_proj", "model.layers.16.mlp.experts.65.down_proj", "model.layers.16.mlp.experts.66.down_proj", "model.layers.16.mlp.experts.67.down_proj", "model.layers.16.mlp.experts.68.down_proj", "model.layers.16.mlp.experts.69.down_proj", "model.layers.16.mlp.experts.70.down_proj", "model.layers.16.mlp.experts.71.down_proj", "model.layers.16.mlp.experts.72.down_proj", "model.layers.16.mlp.experts.73.down_proj", "model.layers.16.mlp.experts.74.down_proj", "model.layers.16.mlp.experts.75.down_proj", "model.layers.16.mlp.experts.76.down_proj", "model.layers.16.mlp.experts.77.down_proj", "model.layers.16.mlp.experts.78.down_proj", "model.layers.16.mlp.experts.79.down_proj", "model.layers.16.mlp.experts.80.down_proj", "model.layers.16.mlp.experts.81.down_proj", "model.layers.16.mlp.experts.82.down_proj", "model.layers.16.mlp.experts.83.down_proj", "model.layers.16.mlp.experts.84.down_proj", "model.layers.16.mlp.experts.85.down_proj", "model.layers.16.mlp.experts.86.down_proj", "model.layers.16.mlp.experts.87.down_proj", "model.layers.16.mlp.experts.88.down_proj", "model.layers.16.mlp.experts.89.down_proj", "model.layers.16.mlp.experts.90.down_proj", "model.layers.16.mlp.experts.91.down_proj", "model.layers.16.mlp.experts.92.down_proj", "model.layers.16.mlp.experts.93.down_proj", "model.layers.16.mlp.experts.94.down_proj", "model.layers.16.mlp.experts.95.down_proj", "model.layers.16.mlp.experts.96.down_proj", "model.layers.16.mlp.experts.97.down_proj", "model.layers.16.mlp.experts.98.down_proj", "model.layers.16.mlp.experts.99.down_proj", "model.layers.16.mlp.experts.100.down_proj", "model.layers.16.mlp.experts.101.down_proj", "model.layers.16.mlp.experts.102.down_proj", "model.layers.16.mlp.experts.103.down_proj", "model.layers.16.mlp.experts.104.down_proj", "model.layers.16.mlp.experts.105.down_proj", "model.layers.16.mlp.experts.106.down_proj", "model.layers.16.mlp.experts.107.down_proj", "model.layers.16.mlp.experts.108.down_proj", "model.layers.16.mlp.experts.109.down_proj", "model.layers.16.mlp.experts.110.down_proj", "model.layers.16.mlp.experts.111.down_proj", "model.layers.16.mlp.experts.112.down_proj", "model.layers.16.mlp.experts.113.down_proj", "model.layers.16.mlp.experts.114.down_proj", "model.layers.16.mlp.experts.115.down_proj", "model.layers.16.mlp.experts.116.down_proj", "model.layers.16.mlp.experts.117.down_proj", "model.layers.16.mlp.experts.118.down_proj", "model.layers.16.mlp.experts.119.down_proj", "model.layers.16.mlp.experts.120.down_proj", "model.layers.16.mlp.experts.121.down_proj", "model.layers.16.mlp.experts.122.down_proj", "model.layers.16.mlp.experts.123.down_proj", "model.layers.16.mlp.experts.124.down_proj", "model.layers.16.mlp.experts.125.down_proj", "model.layers.16.mlp.experts.126.down_proj", "model.layers.16.mlp.experts.127.down_proj", "model.layers.16.mlp.experts.128.down_proj", "model.layers.16.mlp.experts.129.down_proj", "model.layers.16.mlp.experts.130.down_proj", "model.layers.16.mlp.experts.131.down_proj", "model.layers.16.mlp.experts.132.down_proj", "model.layers.16.mlp.experts.133.down_proj", "model.layers.16.mlp.experts.134.down_proj", "model.layers.16.mlp.experts.135.down_proj", "model.layers.16.mlp.experts.136.down_proj", "model.layers.16.mlp.experts.137.down_proj", "model.layers.16.mlp.experts.138.down_proj", "model.layers.16.mlp.experts.139.down_proj", "model.layers.16.mlp.experts.140.down_proj", "model.layers.16.mlp.experts.141.down_proj", "model.layers.16.mlp.experts.142.down_proj", "model.layers.16.mlp.experts.143.down_proj", "model.layers.16.mlp.experts.144.down_proj", "model.layers.16.mlp.experts.145.down_proj", "model.layers.16.mlp.experts.146.down_proj", "model.layers.16.mlp.experts.147.down_proj", "model.layers.16.mlp.experts.148.down_proj", "model.layers.16.mlp.experts.149.down_proj", "model.layers.16.mlp.experts.150.down_proj", "model.layers.16.mlp.experts.151.down_proj", "model.layers.16.mlp.experts.152.down_proj", "model.layers.16.mlp.experts.153.down_proj", "model.layers.16.mlp.experts.154.down_proj", "model.layers.16.mlp.experts.155.down_proj", "model.layers.16.mlp.experts.156.down_proj", "model.layers.16.mlp.experts.157.down_proj", "model.layers.16.mlp.experts.158.down_proj", "model.layers.16.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -4.4651358621194525e-06, "dbits": 2516582400 } ] }, { "idx": 99, "layers": [ "model.layers.17.self_attn.q_proj" ], "candidates": [ { "dkld": 1.6354714171029714e-05, "dbits": 125829120 } ] }, { "idx": 100, "layers": [ "model.layers.17.self_attn.k_proj", "model.layers.17.self_attn.v_proj" ], "candidates": [ { "dkld": -2.827949356287761e-06, "dbits": 20971520 } ] }, { "idx": 101, "layers": [ "model.layers.17.self_attn.o_proj" ], "candidates": [ { "dkld": 1.3874896103516134e-05, "dbits": 125829120 } ] }, { "idx": 102, "layers": [ "model.layers.17.mlp.shared_experts.gate_proj", "model.layers.17.mlp.shared_experts.up_proj", "model.layers.17.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.4623504830524239e-05, "dbits": 47185920 } ] }, { "idx": 103, "layers": [ "model.layers.17.mlp.experts.0.gate_proj", "model.layers.17.mlp.experts.1.gate_proj", "model.layers.17.mlp.experts.2.gate_proj", "model.layers.17.mlp.experts.3.gate_proj", "model.layers.17.mlp.experts.4.gate_proj", "model.layers.17.mlp.experts.5.gate_proj", "model.layers.17.mlp.experts.6.gate_proj", "model.layers.17.mlp.experts.7.gate_proj", "model.layers.17.mlp.experts.8.gate_proj", "model.layers.17.mlp.experts.9.gate_proj", "model.layers.17.mlp.experts.10.gate_proj", "model.layers.17.mlp.experts.11.gate_proj", "model.layers.17.mlp.experts.12.gate_proj", "model.layers.17.mlp.experts.13.gate_proj", "model.layers.17.mlp.experts.14.gate_proj", "model.layers.17.mlp.experts.15.gate_proj", "model.layers.17.mlp.experts.16.gate_proj", "model.layers.17.mlp.experts.17.gate_proj", "model.layers.17.mlp.experts.18.gate_proj", "model.layers.17.mlp.experts.19.gate_proj", "model.layers.17.mlp.experts.20.gate_proj", "model.layers.17.mlp.experts.21.gate_proj", "model.layers.17.mlp.experts.22.gate_proj", "model.layers.17.mlp.experts.23.gate_proj", "model.layers.17.mlp.experts.24.gate_proj", "model.layers.17.mlp.experts.25.gate_proj", "model.layers.17.mlp.experts.26.gate_proj", "model.layers.17.mlp.experts.27.gate_proj", "model.layers.17.mlp.experts.28.gate_proj", "model.layers.17.mlp.experts.29.gate_proj", "model.layers.17.mlp.experts.30.gate_proj", "model.layers.17.mlp.experts.31.gate_proj", "model.layers.17.mlp.experts.32.gate_proj", "model.layers.17.mlp.experts.33.gate_proj", "model.layers.17.mlp.experts.34.gate_proj", "model.layers.17.mlp.experts.35.gate_proj", "model.layers.17.mlp.experts.36.gate_proj", "model.layers.17.mlp.experts.37.gate_proj", "model.layers.17.mlp.experts.38.gate_proj", "model.layers.17.mlp.experts.39.gate_proj", "model.layers.17.mlp.experts.40.gate_proj", "model.layers.17.mlp.experts.41.gate_proj", "model.layers.17.mlp.experts.42.gate_proj", "model.layers.17.mlp.experts.43.gate_proj", "model.layers.17.mlp.experts.44.gate_proj", "model.layers.17.mlp.experts.45.gate_proj", "model.layers.17.mlp.experts.46.gate_proj", "model.layers.17.mlp.experts.47.gate_proj", "model.layers.17.mlp.experts.48.gate_proj", "model.layers.17.mlp.experts.49.gate_proj", "model.layers.17.mlp.experts.50.gate_proj", "model.layers.17.mlp.experts.51.gate_proj", "model.layers.17.mlp.experts.52.gate_proj", "model.layers.17.mlp.experts.53.gate_proj", "model.layers.17.mlp.experts.54.gate_proj", "model.layers.17.mlp.experts.55.gate_proj", "model.layers.17.mlp.experts.56.gate_proj", "model.layers.17.mlp.experts.57.gate_proj", "model.layers.17.mlp.experts.58.gate_proj", "model.layers.17.mlp.experts.59.gate_proj", "model.layers.17.mlp.experts.60.gate_proj", "model.layers.17.mlp.experts.61.gate_proj", "model.layers.17.mlp.experts.62.gate_proj", "model.layers.17.mlp.experts.63.gate_proj", "model.layers.17.mlp.experts.64.gate_proj", "model.layers.17.mlp.experts.65.gate_proj", "model.layers.17.mlp.experts.66.gate_proj", "model.layers.17.mlp.experts.67.gate_proj", "model.layers.17.mlp.experts.68.gate_proj", "model.layers.17.mlp.experts.69.gate_proj", "model.layers.17.mlp.experts.70.gate_proj", "model.layers.17.mlp.experts.71.gate_proj", "model.layers.17.mlp.experts.72.gate_proj", "model.layers.17.mlp.experts.73.gate_proj", "model.layers.17.mlp.experts.74.gate_proj", "model.layers.17.mlp.experts.75.gate_proj", "model.layers.17.mlp.experts.76.gate_proj", "model.layers.17.mlp.experts.77.gate_proj", "model.layers.17.mlp.experts.78.gate_proj", "model.layers.17.mlp.experts.79.gate_proj", "model.layers.17.mlp.experts.80.gate_proj", "model.layers.17.mlp.experts.81.gate_proj", "model.layers.17.mlp.experts.82.gate_proj", "model.layers.17.mlp.experts.83.gate_proj", "model.layers.17.mlp.experts.84.gate_proj", "model.layers.17.mlp.experts.85.gate_proj", "model.layers.17.mlp.experts.86.gate_proj", "model.layers.17.mlp.experts.87.gate_proj", "model.layers.17.mlp.experts.88.gate_proj", "model.layers.17.mlp.experts.89.gate_proj", "model.layers.17.mlp.experts.90.gate_proj", "model.layers.17.mlp.experts.91.gate_proj", "model.layers.17.mlp.experts.92.gate_proj", "model.layers.17.mlp.experts.93.gate_proj", "model.layers.17.mlp.experts.94.gate_proj", "model.layers.17.mlp.experts.95.gate_proj", "model.layers.17.mlp.experts.96.gate_proj", "model.layers.17.mlp.experts.97.gate_proj", "model.layers.17.mlp.experts.98.gate_proj", "model.layers.17.mlp.experts.99.gate_proj", "model.layers.17.mlp.experts.100.gate_proj", "model.layers.17.mlp.experts.101.gate_proj", "model.layers.17.mlp.experts.102.gate_proj", "model.layers.17.mlp.experts.103.gate_proj", "model.layers.17.mlp.experts.104.gate_proj", "model.layers.17.mlp.experts.105.gate_proj", "model.layers.17.mlp.experts.106.gate_proj", "model.layers.17.mlp.experts.107.gate_proj", "model.layers.17.mlp.experts.108.gate_proj", "model.layers.17.mlp.experts.109.gate_proj", "model.layers.17.mlp.experts.110.gate_proj", "model.layers.17.mlp.experts.111.gate_proj", "model.layers.17.mlp.experts.112.gate_proj", "model.layers.17.mlp.experts.113.gate_proj", "model.layers.17.mlp.experts.114.gate_proj", "model.layers.17.mlp.experts.115.gate_proj", "model.layers.17.mlp.experts.116.gate_proj", "model.layers.17.mlp.experts.117.gate_proj", "model.layers.17.mlp.experts.118.gate_proj", "model.layers.17.mlp.experts.119.gate_proj", "model.layers.17.mlp.experts.120.gate_proj", "model.layers.17.mlp.experts.121.gate_proj", "model.layers.17.mlp.experts.122.gate_proj", "model.layers.17.mlp.experts.123.gate_proj", "model.layers.17.mlp.experts.124.gate_proj", "model.layers.17.mlp.experts.125.gate_proj", "model.layers.17.mlp.experts.126.gate_proj", "model.layers.17.mlp.experts.127.gate_proj", "model.layers.17.mlp.experts.128.gate_proj", "model.layers.17.mlp.experts.129.gate_proj", "model.layers.17.mlp.experts.130.gate_proj", "model.layers.17.mlp.experts.131.gate_proj", "model.layers.17.mlp.experts.132.gate_proj", "model.layers.17.mlp.experts.133.gate_proj", "model.layers.17.mlp.experts.134.gate_proj", "model.layers.17.mlp.experts.135.gate_proj", "model.layers.17.mlp.experts.136.gate_proj", "model.layers.17.mlp.experts.137.gate_proj", "model.layers.17.mlp.experts.138.gate_proj", "model.layers.17.mlp.experts.139.gate_proj", "model.layers.17.mlp.experts.140.gate_proj", "model.layers.17.mlp.experts.141.gate_proj", "model.layers.17.mlp.experts.142.gate_proj", "model.layers.17.mlp.experts.143.gate_proj", "model.layers.17.mlp.experts.144.gate_proj", "model.layers.17.mlp.experts.145.gate_proj", "model.layers.17.mlp.experts.146.gate_proj", "model.layers.17.mlp.experts.147.gate_proj", "model.layers.17.mlp.experts.148.gate_proj", "model.layers.17.mlp.experts.149.gate_proj", "model.layers.17.mlp.experts.150.gate_proj", "model.layers.17.mlp.experts.151.gate_proj", "model.layers.17.mlp.experts.152.gate_proj", "model.layers.17.mlp.experts.153.gate_proj", "model.layers.17.mlp.experts.154.gate_proj", "model.layers.17.mlp.experts.155.gate_proj", "model.layers.17.mlp.experts.156.gate_proj", "model.layers.17.mlp.experts.157.gate_proj", "model.layers.17.mlp.experts.158.gate_proj", "model.layers.17.mlp.experts.159.gate_proj", "model.layers.17.mlp.experts.0.up_proj", "model.layers.17.mlp.experts.1.up_proj", "model.layers.17.mlp.experts.2.up_proj", "model.layers.17.mlp.experts.3.up_proj", "model.layers.17.mlp.experts.4.up_proj", "model.layers.17.mlp.experts.5.up_proj", "model.layers.17.mlp.experts.6.up_proj", "model.layers.17.mlp.experts.7.up_proj", "model.layers.17.mlp.experts.8.up_proj", "model.layers.17.mlp.experts.9.up_proj", "model.layers.17.mlp.experts.10.up_proj", "model.layers.17.mlp.experts.11.up_proj", "model.layers.17.mlp.experts.12.up_proj", "model.layers.17.mlp.experts.13.up_proj", "model.layers.17.mlp.experts.14.up_proj", "model.layers.17.mlp.experts.15.up_proj", "model.layers.17.mlp.experts.16.up_proj", "model.layers.17.mlp.experts.17.up_proj", "model.layers.17.mlp.experts.18.up_proj", "model.layers.17.mlp.experts.19.up_proj", "model.layers.17.mlp.experts.20.up_proj", "model.layers.17.mlp.experts.21.up_proj", "model.layers.17.mlp.experts.22.up_proj", "model.layers.17.mlp.experts.23.up_proj", "model.layers.17.mlp.experts.24.up_proj", "model.layers.17.mlp.experts.25.up_proj", "model.layers.17.mlp.experts.26.up_proj", "model.layers.17.mlp.experts.27.up_proj", "model.layers.17.mlp.experts.28.up_proj", "model.layers.17.mlp.experts.29.up_proj", "model.layers.17.mlp.experts.30.up_proj", "model.layers.17.mlp.experts.31.up_proj", "model.layers.17.mlp.experts.32.up_proj", "model.layers.17.mlp.experts.33.up_proj", "model.layers.17.mlp.experts.34.up_proj", "model.layers.17.mlp.experts.35.up_proj", "model.layers.17.mlp.experts.36.up_proj", "model.layers.17.mlp.experts.37.up_proj", "model.layers.17.mlp.experts.38.up_proj", "model.layers.17.mlp.experts.39.up_proj", "model.layers.17.mlp.experts.40.up_proj", "model.layers.17.mlp.experts.41.up_proj", "model.layers.17.mlp.experts.42.up_proj", "model.layers.17.mlp.experts.43.up_proj", "model.layers.17.mlp.experts.44.up_proj", "model.layers.17.mlp.experts.45.up_proj", "model.layers.17.mlp.experts.46.up_proj", "model.layers.17.mlp.experts.47.up_proj", "model.layers.17.mlp.experts.48.up_proj", "model.layers.17.mlp.experts.49.up_proj", "model.layers.17.mlp.experts.50.up_proj", "model.layers.17.mlp.experts.51.up_proj", "model.layers.17.mlp.experts.52.up_proj", "model.layers.17.mlp.experts.53.up_proj", "model.layers.17.mlp.experts.54.up_proj", "model.layers.17.mlp.experts.55.up_proj", "model.layers.17.mlp.experts.56.up_proj", "model.layers.17.mlp.experts.57.up_proj", "model.layers.17.mlp.experts.58.up_proj", "model.layers.17.mlp.experts.59.up_proj", "model.layers.17.mlp.experts.60.up_proj", "model.layers.17.mlp.experts.61.up_proj", "model.layers.17.mlp.experts.62.up_proj", "model.layers.17.mlp.experts.63.up_proj", "model.layers.17.mlp.experts.64.up_proj", "model.layers.17.mlp.experts.65.up_proj", "model.layers.17.mlp.experts.66.up_proj", "model.layers.17.mlp.experts.67.up_proj", "model.layers.17.mlp.experts.68.up_proj", "model.layers.17.mlp.experts.69.up_proj", "model.layers.17.mlp.experts.70.up_proj", "model.layers.17.mlp.experts.71.up_proj", "model.layers.17.mlp.experts.72.up_proj", "model.layers.17.mlp.experts.73.up_proj", "model.layers.17.mlp.experts.74.up_proj", "model.layers.17.mlp.experts.75.up_proj", "model.layers.17.mlp.experts.76.up_proj", "model.layers.17.mlp.experts.77.up_proj", "model.layers.17.mlp.experts.78.up_proj", "model.layers.17.mlp.experts.79.up_proj", "model.layers.17.mlp.experts.80.up_proj", "model.layers.17.mlp.experts.81.up_proj", "model.layers.17.mlp.experts.82.up_proj", "model.layers.17.mlp.experts.83.up_proj", "model.layers.17.mlp.experts.84.up_proj", "model.layers.17.mlp.experts.85.up_proj", "model.layers.17.mlp.experts.86.up_proj", "model.layers.17.mlp.experts.87.up_proj", "model.layers.17.mlp.experts.88.up_proj", "model.layers.17.mlp.experts.89.up_proj", "model.layers.17.mlp.experts.90.up_proj", "model.layers.17.mlp.experts.91.up_proj", "model.layers.17.mlp.experts.92.up_proj", "model.layers.17.mlp.experts.93.up_proj", "model.layers.17.mlp.experts.94.up_proj", "model.layers.17.mlp.experts.95.up_proj", "model.layers.17.mlp.experts.96.up_proj", "model.layers.17.mlp.experts.97.up_proj", "model.layers.17.mlp.experts.98.up_proj", "model.layers.17.mlp.experts.99.up_proj", "model.layers.17.mlp.experts.100.up_proj", "model.layers.17.mlp.experts.101.up_proj", "model.layers.17.mlp.experts.102.up_proj", "model.layers.17.mlp.experts.103.up_proj", "model.layers.17.mlp.experts.104.up_proj", "model.layers.17.mlp.experts.105.up_proj", "model.layers.17.mlp.experts.106.up_proj", "model.layers.17.mlp.experts.107.up_proj", "model.layers.17.mlp.experts.108.up_proj", "model.layers.17.mlp.experts.109.up_proj", "model.layers.17.mlp.experts.110.up_proj", "model.layers.17.mlp.experts.111.up_proj", "model.layers.17.mlp.experts.112.up_proj", "model.layers.17.mlp.experts.113.up_proj", "model.layers.17.mlp.experts.114.up_proj", "model.layers.17.mlp.experts.115.up_proj", "model.layers.17.mlp.experts.116.up_proj", "model.layers.17.mlp.experts.117.up_proj", "model.layers.17.mlp.experts.118.up_proj", "model.layers.17.mlp.experts.119.up_proj", "model.layers.17.mlp.experts.120.up_proj", "model.layers.17.mlp.experts.121.up_proj", "model.layers.17.mlp.experts.122.up_proj", "model.layers.17.mlp.experts.123.up_proj", "model.layers.17.mlp.experts.124.up_proj", "model.layers.17.mlp.experts.125.up_proj", "model.layers.17.mlp.experts.126.up_proj", "model.layers.17.mlp.experts.127.up_proj", "model.layers.17.mlp.experts.128.up_proj", "model.layers.17.mlp.experts.129.up_proj", "model.layers.17.mlp.experts.130.up_proj", "model.layers.17.mlp.experts.131.up_proj", "model.layers.17.mlp.experts.132.up_proj", "model.layers.17.mlp.experts.133.up_proj", "model.layers.17.mlp.experts.134.up_proj", "model.layers.17.mlp.experts.135.up_proj", "model.layers.17.mlp.experts.136.up_proj", "model.layers.17.mlp.experts.137.up_proj", "model.layers.17.mlp.experts.138.up_proj", "model.layers.17.mlp.experts.139.up_proj", "model.layers.17.mlp.experts.140.up_proj", "model.layers.17.mlp.experts.141.up_proj", "model.layers.17.mlp.experts.142.up_proj", "model.layers.17.mlp.experts.143.up_proj", "model.layers.17.mlp.experts.144.up_proj", "model.layers.17.mlp.experts.145.up_proj", "model.layers.17.mlp.experts.146.up_proj", "model.layers.17.mlp.experts.147.up_proj", "model.layers.17.mlp.experts.148.up_proj", "model.layers.17.mlp.experts.149.up_proj", "model.layers.17.mlp.experts.150.up_proj", "model.layers.17.mlp.experts.151.up_proj", "model.layers.17.mlp.experts.152.up_proj", "model.layers.17.mlp.experts.153.up_proj", "model.layers.17.mlp.experts.154.up_proj", "model.layers.17.mlp.experts.155.up_proj", "model.layers.17.mlp.experts.156.up_proj", "model.layers.17.mlp.experts.157.up_proj", "model.layers.17.mlp.experts.158.up_proj", "model.layers.17.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.571019529365001e-05, "dbits": 5033164800 } ] }, { "idx": 104, "layers": [ "model.layers.17.mlp.experts.0.down_proj", "model.layers.17.mlp.experts.1.down_proj", "model.layers.17.mlp.experts.2.down_proj", "model.layers.17.mlp.experts.3.down_proj", "model.layers.17.mlp.experts.4.down_proj", "model.layers.17.mlp.experts.5.down_proj", "model.layers.17.mlp.experts.6.down_proj", "model.layers.17.mlp.experts.7.down_proj", "model.layers.17.mlp.experts.8.down_proj", "model.layers.17.mlp.experts.9.down_proj", "model.layers.17.mlp.experts.10.down_proj", "model.layers.17.mlp.experts.11.down_proj", "model.layers.17.mlp.experts.12.down_proj", "model.layers.17.mlp.experts.13.down_proj", "model.layers.17.mlp.experts.14.down_proj", "model.layers.17.mlp.experts.15.down_proj", "model.layers.17.mlp.experts.16.down_proj", "model.layers.17.mlp.experts.17.down_proj", "model.layers.17.mlp.experts.18.down_proj", "model.layers.17.mlp.experts.19.down_proj", "model.layers.17.mlp.experts.20.down_proj", "model.layers.17.mlp.experts.21.down_proj", "model.layers.17.mlp.experts.22.down_proj", "model.layers.17.mlp.experts.23.down_proj", "model.layers.17.mlp.experts.24.down_proj", "model.layers.17.mlp.experts.25.down_proj", "model.layers.17.mlp.experts.26.down_proj", "model.layers.17.mlp.experts.27.down_proj", "model.layers.17.mlp.experts.28.down_proj", "model.layers.17.mlp.experts.29.down_proj", "model.layers.17.mlp.experts.30.down_proj", "model.layers.17.mlp.experts.31.down_proj", "model.layers.17.mlp.experts.32.down_proj", "model.layers.17.mlp.experts.33.down_proj", "model.layers.17.mlp.experts.34.down_proj", "model.layers.17.mlp.experts.35.down_proj", "model.layers.17.mlp.experts.36.down_proj", "model.layers.17.mlp.experts.37.down_proj", "model.layers.17.mlp.experts.38.down_proj", "model.layers.17.mlp.experts.39.down_proj", "model.layers.17.mlp.experts.40.down_proj", "model.layers.17.mlp.experts.41.down_proj", "model.layers.17.mlp.experts.42.down_proj", "model.layers.17.mlp.experts.43.down_proj", "model.layers.17.mlp.experts.44.down_proj", "model.layers.17.mlp.experts.45.down_proj", "model.layers.17.mlp.experts.46.down_proj", "model.layers.17.mlp.experts.47.down_proj", "model.layers.17.mlp.experts.48.down_proj", "model.layers.17.mlp.experts.49.down_proj", "model.layers.17.mlp.experts.50.down_proj", "model.layers.17.mlp.experts.51.down_proj", "model.layers.17.mlp.experts.52.down_proj", "model.layers.17.mlp.experts.53.down_proj", "model.layers.17.mlp.experts.54.down_proj", "model.layers.17.mlp.experts.55.down_proj", "model.layers.17.mlp.experts.56.down_proj", "model.layers.17.mlp.experts.57.down_proj", "model.layers.17.mlp.experts.58.down_proj", "model.layers.17.mlp.experts.59.down_proj", "model.layers.17.mlp.experts.60.down_proj", "model.layers.17.mlp.experts.61.down_proj", "model.layers.17.mlp.experts.62.down_proj", "model.layers.17.mlp.experts.63.down_proj", "model.layers.17.mlp.experts.64.down_proj", "model.layers.17.mlp.experts.65.down_proj", "model.layers.17.mlp.experts.66.down_proj", "model.layers.17.mlp.experts.67.down_proj", "model.layers.17.mlp.experts.68.down_proj", "model.layers.17.mlp.experts.69.down_proj", "model.layers.17.mlp.experts.70.down_proj", "model.layers.17.mlp.experts.71.down_proj", "model.layers.17.mlp.experts.72.down_proj", "model.layers.17.mlp.experts.73.down_proj", "model.layers.17.mlp.experts.74.down_proj", "model.layers.17.mlp.experts.75.down_proj", "model.layers.17.mlp.experts.76.down_proj", "model.layers.17.mlp.experts.77.down_proj", "model.layers.17.mlp.experts.78.down_proj", "model.layers.17.mlp.experts.79.down_proj", "model.layers.17.mlp.experts.80.down_proj", "model.layers.17.mlp.experts.81.down_proj", "model.layers.17.mlp.experts.82.down_proj", "model.layers.17.mlp.experts.83.down_proj", "model.layers.17.mlp.experts.84.down_proj", "model.layers.17.mlp.experts.85.down_proj", "model.layers.17.mlp.experts.86.down_proj", "model.layers.17.mlp.experts.87.down_proj", "model.layers.17.mlp.experts.88.down_proj", "model.layers.17.mlp.experts.89.down_proj", "model.layers.17.mlp.experts.90.down_proj", "model.layers.17.mlp.experts.91.down_proj", "model.layers.17.mlp.experts.92.down_proj", "model.layers.17.mlp.experts.93.down_proj", "model.layers.17.mlp.experts.94.down_proj", "model.layers.17.mlp.experts.95.down_proj", "model.layers.17.mlp.experts.96.down_proj", "model.layers.17.mlp.experts.97.down_proj", "model.layers.17.mlp.experts.98.down_proj", "model.layers.17.mlp.experts.99.down_proj", "model.layers.17.mlp.experts.100.down_proj", "model.layers.17.mlp.experts.101.down_proj", "model.layers.17.mlp.experts.102.down_proj", "model.layers.17.mlp.experts.103.down_proj", "model.layers.17.mlp.experts.104.down_proj", "model.layers.17.mlp.experts.105.down_proj", "model.layers.17.mlp.experts.106.down_proj", "model.layers.17.mlp.experts.107.down_proj", "model.layers.17.mlp.experts.108.down_proj", "model.layers.17.mlp.experts.109.down_proj", "model.layers.17.mlp.experts.110.down_proj", "model.layers.17.mlp.experts.111.down_proj", "model.layers.17.mlp.experts.112.down_proj", "model.layers.17.mlp.experts.113.down_proj", "model.layers.17.mlp.experts.114.down_proj", "model.layers.17.mlp.experts.115.down_proj", "model.layers.17.mlp.experts.116.down_proj", "model.layers.17.mlp.experts.117.down_proj", "model.layers.17.mlp.experts.118.down_proj", "model.layers.17.mlp.experts.119.down_proj", "model.layers.17.mlp.experts.120.down_proj", "model.layers.17.mlp.experts.121.down_proj", "model.layers.17.mlp.experts.122.down_proj", "model.layers.17.mlp.experts.123.down_proj", "model.layers.17.mlp.experts.124.down_proj", "model.layers.17.mlp.experts.125.down_proj", "model.layers.17.mlp.experts.126.down_proj", "model.layers.17.mlp.experts.127.down_proj", "model.layers.17.mlp.experts.128.down_proj", "model.layers.17.mlp.experts.129.down_proj", "model.layers.17.mlp.experts.130.down_proj", "model.layers.17.mlp.experts.131.down_proj", "model.layers.17.mlp.experts.132.down_proj", "model.layers.17.mlp.experts.133.down_proj", "model.layers.17.mlp.experts.134.down_proj", "model.layers.17.mlp.experts.135.down_proj", "model.layers.17.mlp.experts.136.down_proj", "model.layers.17.mlp.experts.137.down_proj", "model.layers.17.mlp.experts.138.down_proj", "model.layers.17.mlp.experts.139.down_proj", "model.layers.17.mlp.experts.140.down_proj", "model.layers.17.mlp.experts.141.down_proj", "model.layers.17.mlp.experts.142.down_proj", "model.layers.17.mlp.experts.143.down_proj", "model.layers.17.mlp.experts.144.down_proj", "model.layers.17.mlp.experts.145.down_proj", "model.layers.17.mlp.experts.146.down_proj", "model.layers.17.mlp.experts.147.down_proj", "model.layers.17.mlp.experts.148.down_proj", "model.layers.17.mlp.experts.149.down_proj", "model.layers.17.mlp.experts.150.down_proj", "model.layers.17.mlp.experts.151.down_proj", "model.layers.17.mlp.experts.152.down_proj", "model.layers.17.mlp.experts.153.down_proj", "model.layers.17.mlp.experts.154.down_proj", "model.layers.17.mlp.experts.155.down_proj", "model.layers.17.mlp.experts.156.down_proj", "model.layers.17.mlp.experts.157.down_proj", "model.layers.17.mlp.experts.158.down_proj", "model.layers.17.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.2249418068677209e-05, "dbits": 2516582400 } ] }, { "idx": 105, "layers": [ "model.layers.18.self_attn.q_proj" ], "candidates": [ { "dkld": 2.798551577143371e-06, "dbits": 125829120 } ] }, { "idx": 106, "layers": [ "model.layers.18.self_attn.k_proj", "model.layers.18.self_attn.v_proj" ], "candidates": [ { "dkld": 3.7205230910330794e-06, "dbits": 20971520 } ] }, { "idx": 107, "layers": [ "model.layers.18.self_attn.o_proj" ], "candidates": [ { "dkld": -1.220099511556349e-06, "dbits": 125829120 } ] }, { "idx": 108, "layers": [ "model.layers.18.mlp.shared_experts.gate_proj", "model.layers.18.mlp.shared_experts.up_proj", "model.layers.18.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.424254780635242e-05, "dbits": 47185920 } ] }, { "idx": 109, "layers": [ "model.layers.18.mlp.experts.0.gate_proj", "model.layers.18.mlp.experts.1.gate_proj", "model.layers.18.mlp.experts.2.gate_proj", "model.layers.18.mlp.experts.3.gate_proj", "model.layers.18.mlp.experts.4.gate_proj", "model.layers.18.mlp.experts.5.gate_proj", "model.layers.18.mlp.experts.6.gate_proj", "model.layers.18.mlp.experts.7.gate_proj", "model.layers.18.mlp.experts.8.gate_proj", "model.layers.18.mlp.experts.9.gate_proj", "model.layers.18.mlp.experts.10.gate_proj", "model.layers.18.mlp.experts.11.gate_proj", "model.layers.18.mlp.experts.12.gate_proj", "model.layers.18.mlp.experts.13.gate_proj", "model.layers.18.mlp.experts.14.gate_proj", "model.layers.18.mlp.experts.15.gate_proj", "model.layers.18.mlp.experts.16.gate_proj", "model.layers.18.mlp.experts.17.gate_proj", "model.layers.18.mlp.experts.18.gate_proj", "model.layers.18.mlp.experts.19.gate_proj", "model.layers.18.mlp.experts.20.gate_proj", "model.layers.18.mlp.experts.21.gate_proj", "model.layers.18.mlp.experts.22.gate_proj", "model.layers.18.mlp.experts.23.gate_proj", "model.layers.18.mlp.experts.24.gate_proj", "model.layers.18.mlp.experts.25.gate_proj", "model.layers.18.mlp.experts.26.gate_proj", "model.layers.18.mlp.experts.27.gate_proj", "model.layers.18.mlp.experts.28.gate_proj", "model.layers.18.mlp.experts.29.gate_proj", "model.layers.18.mlp.experts.30.gate_proj", "model.layers.18.mlp.experts.31.gate_proj", "model.layers.18.mlp.experts.32.gate_proj", "model.layers.18.mlp.experts.33.gate_proj", "model.layers.18.mlp.experts.34.gate_proj", "model.layers.18.mlp.experts.35.gate_proj", "model.layers.18.mlp.experts.36.gate_proj", "model.layers.18.mlp.experts.37.gate_proj", "model.layers.18.mlp.experts.38.gate_proj", "model.layers.18.mlp.experts.39.gate_proj", "model.layers.18.mlp.experts.40.gate_proj", "model.layers.18.mlp.experts.41.gate_proj", "model.layers.18.mlp.experts.42.gate_proj", "model.layers.18.mlp.experts.43.gate_proj", "model.layers.18.mlp.experts.44.gate_proj", "model.layers.18.mlp.experts.45.gate_proj", "model.layers.18.mlp.experts.46.gate_proj", "model.layers.18.mlp.experts.47.gate_proj", "model.layers.18.mlp.experts.48.gate_proj", "model.layers.18.mlp.experts.49.gate_proj", "model.layers.18.mlp.experts.50.gate_proj", "model.layers.18.mlp.experts.51.gate_proj", "model.layers.18.mlp.experts.52.gate_proj", "model.layers.18.mlp.experts.53.gate_proj", "model.layers.18.mlp.experts.54.gate_proj", "model.layers.18.mlp.experts.55.gate_proj", "model.layers.18.mlp.experts.56.gate_proj", "model.layers.18.mlp.experts.57.gate_proj", "model.layers.18.mlp.experts.58.gate_proj", "model.layers.18.mlp.experts.59.gate_proj", "model.layers.18.mlp.experts.60.gate_proj", "model.layers.18.mlp.experts.61.gate_proj", "model.layers.18.mlp.experts.62.gate_proj", "model.layers.18.mlp.experts.63.gate_proj", "model.layers.18.mlp.experts.64.gate_proj", "model.layers.18.mlp.experts.65.gate_proj", "model.layers.18.mlp.experts.66.gate_proj", "model.layers.18.mlp.experts.67.gate_proj", "model.layers.18.mlp.experts.68.gate_proj", "model.layers.18.mlp.experts.69.gate_proj", "model.layers.18.mlp.experts.70.gate_proj", "model.layers.18.mlp.experts.71.gate_proj", "model.layers.18.mlp.experts.72.gate_proj", "model.layers.18.mlp.experts.73.gate_proj", "model.layers.18.mlp.experts.74.gate_proj", "model.layers.18.mlp.experts.75.gate_proj", "model.layers.18.mlp.experts.76.gate_proj", "model.layers.18.mlp.experts.77.gate_proj", "model.layers.18.mlp.experts.78.gate_proj", "model.layers.18.mlp.experts.79.gate_proj", "model.layers.18.mlp.experts.80.gate_proj", "model.layers.18.mlp.experts.81.gate_proj", "model.layers.18.mlp.experts.82.gate_proj", "model.layers.18.mlp.experts.83.gate_proj", "model.layers.18.mlp.experts.84.gate_proj", "model.layers.18.mlp.experts.85.gate_proj", "model.layers.18.mlp.experts.86.gate_proj", "model.layers.18.mlp.experts.87.gate_proj", "model.layers.18.mlp.experts.88.gate_proj", "model.layers.18.mlp.experts.89.gate_proj", "model.layers.18.mlp.experts.90.gate_proj", "model.layers.18.mlp.experts.91.gate_proj", "model.layers.18.mlp.experts.92.gate_proj", "model.layers.18.mlp.experts.93.gate_proj", "model.layers.18.mlp.experts.94.gate_proj", "model.layers.18.mlp.experts.95.gate_proj", "model.layers.18.mlp.experts.96.gate_proj", "model.layers.18.mlp.experts.97.gate_proj", "model.layers.18.mlp.experts.98.gate_proj", "model.layers.18.mlp.experts.99.gate_proj", "model.layers.18.mlp.experts.100.gate_proj", "model.layers.18.mlp.experts.101.gate_proj", "model.layers.18.mlp.experts.102.gate_proj", "model.layers.18.mlp.experts.103.gate_proj", "model.layers.18.mlp.experts.104.gate_proj", "model.layers.18.mlp.experts.105.gate_proj", "model.layers.18.mlp.experts.106.gate_proj", "model.layers.18.mlp.experts.107.gate_proj", "model.layers.18.mlp.experts.108.gate_proj", "model.layers.18.mlp.experts.109.gate_proj", "model.layers.18.mlp.experts.110.gate_proj", "model.layers.18.mlp.experts.111.gate_proj", "model.layers.18.mlp.experts.112.gate_proj", "model.layers.18.mlp.experts.113.gate_proj", "model.layers.18.mlp.experts.114.gate_proj", "model.layers.18.mlp.experts.115.gate_proj", "model.layers.18.mlp.experts.116.gate_proj", "model.layers.18.mlp.experts.117.gate_proj", "model.layers.18.mlp.experts.118.gate_proj", "model.layers.18.mlp.experts.119.gate_proj", "model.layers.18.mlp.experts.120.gate_proj", "model.layers.18.mlp.experts.121.gate_proj", "model.layers.18.mlp.experts.122.gate_proj", "model.layers.18.mlp.experts.123.gate_proj", "model.layers.18.mlp.experts.124.gate_proj", "model.layers.18.mlp.experts.125.gate_proj", "model.layers.18.mlp.experts.126.gate_proj", "model.layers.18.mlp.experts.127.gate_proj", "model.layers.18.mlp.experts.128.gate_proj", "model.layers.18.mlp.experts.129.gate_proj", "model.layers.18.mlp.experts.130.gate_proj", "model.layers.18.mlp.experts.131.gate_proj", "model.layers.18.mlp.experts.132.gate_proj", "model.layers.18.mlp.experts.133.gate_proj", "model.layers.18.mlp.experts.134.gate_proj", "model.layers.18.mlp.experts.135.gate_proj", "model.layers.18.mlp.experts.136.gate_proj", "model.layers.18.mlp.experts.137.gate_proj", "model.layers.18.mlp.experts.138.gate_proj", "model.layers.18.mlp.experts.139.gate_proj", "model.layers.18.mlp.experts.140.gate_proj", "model.layers.18.mlp.experts.141.gate_proj", "model.layers.18.mlp.experts.142.gate_proj", "model.layers.18.mlp.experts.143.gate_proj", "model.layers.18.mlp.experts.144.gate_proj", "model.layers.18.mlp.experts.145.gate_proj", "model.layers.18.mlp.experts.146.gate_proj", "model.layers.18.mlp.experts.147.gate_proj", "model.layers.18.mlp.experts.148.gate_proj", "model.layers.18.mlp.experts.149.gate_proj", "model.layers.18.mlp.experts.150.gate_proj", "model.layers.18.mlp.experts.151.gate_proj", "model.layers.18.mlp.experts.152.gate_proj", "model.layers.18.mlp.experts.153.gate_proj", "model.layers.18.mlp.experts.154.gate_proj", "model.layers.18.mlp.experts.155.gate_proj", "model.layers.18.mlp.experts.156.gate_proj", "model.layers.18.mlp.experts.157.gate_proj", "model.layers.18.mlp.experts.158.gate_proj", "model.layers.18.mlp.experts.159.gate_proj", "model.layers.18.mlp.experts.0.up_proj", "model.layers.18.mlp.experts.1.up_proj", "model.layers.18.mlp.experts.2.up_proj", "model.layers.18.mlp.experts.3.up_proj", "model.layers.18.mlp.experts.4.up_proj", "model.layers.18.mlp.experts.5.up_proj", "model.layers.18.mlp.experts.6.up_proj", "model.layers.18.mlp.experts.7.up_proj", "model.layers.18.mlp.experts.8.up_proj", "model.layers.18.mlp.experts.9.up_proj", "model.layers.18.mlp.experts.10.up_proj", "model.layers.18.mlp.experts.11.up_proj", "model.layers.18.mlp.experts.12.up_proj", "model.layers.18.mlp.experts.13.up_proj", "model.layers.18.mlp.experts.14.up_proj", "model.layers.18.mlp.experts.15.up_proj", "model.layers.18.mlp.experts.16.up_proj", "model.layers.18.mlp.experts.17.up_proj", "model.layers.18.mlp.experts.18.up_proj", "model.layers.18.mlp.experts.19.up_proj", "model.layers.18.mlp.experts.20.up_proj", "model.layers.18.mlp.experts.21.up_proj", "model.layers.18.mlp.experts.22.up_proj", "model.layers.18.mlp.experts.23.up_proj", "model.layers.18.mlp.experts.24.up_proj", "model.layers.18.mlp.experts.25.up_proj", "model.layers.18.mlp.experts.26.up_proj", "model.layers.18.mlp.experts.27.up_proj", "model.layers.18.mlp.experts.28.up_proj", "model.layers.18.mlp.experts.29.up_proj", "model.layers.18.mlp.experts.30.up_proj", "model.layers.18.mlp.experts.31.up_proj", "model.layers.18.mlp.experts.32.up_proj", "model.layers.18.mlp.experts.33.up_proj", "model.layers.18.mlp.experts.34.up_proj", "model.layers.18.mlp.experts.35.up_proj", "model.layers.18.mlp.experts.36.up_proj", "model.layers.18.mlp.experts.37.up_proj", "model.layers.18.mlp.experts.38.up_proj", "model.layers.18.mlp.experts.39.up_proj", "model.layers.18.mlp.experts.40.up_proj", "model.layers.18.mlp.experts.41.up_proj", "model.layers.18.mlp.experts.42.up_proj", "model.layers.18.mlp.experts.43.up_proj", "model.layers.18.mlp.experts.44.up_proj", "model.layers.18.mlp.experts.45.up_proj", "model.layers.18.mlp.experts.46.up_proj", "model.layers.18.mlp.experts.47.up_proj", "model.layers.18.mlp.experts.48.up_proj", "model.layers.18.mlp.experts.49.up_proj", "model.layers.18.mlp.experts.50.up_proj", "model.layers.18.mlp.experts.51.up_proj", "model.layers.18.mlp.experts.52.up_proj", "model.layers.18.mlp.experts.53.up_proj", "model.layers.18.mlp.experts.54.up_proj", "model.layers.18.mlp.experts.55.up_proj", "model.layers.18.mlp.experts.56.up_proj", "model.layers.18.mlp.experts.57.up_proj", "model.layers.18.mlp.experts.58.up_proj", "model.layers.18.mlp.experts.59.up_proj", "model.layers.18.mlp.experts.60.up_proj", "model.layers.18.mlp.experts.61.up_proj", "model.layers.18.mlp.experts.62.up_proj", "model.layers.18.mlp.experts.63.up_proj", "model.layers.18.mlp.experts.64.up_proj", "model.layers.18.mlp.experts.65.up_proj", "model.layers.18.mlp.experts.66.up_proj", "model.layers.18.mlp.experts.67.up_proj", "model.layers.18.mlp.experts.68.up_proj", "model.layers.18.mlp.experts.69.up_proj", "model.layers.18.mlp.experts.70.up_proj", "model.layers.18.mlp.experts.71.up_proj", "model.layers.18.mlp.experts.72.up_proj", "model.layers.18.mlp.experts.73.up_proj", "model.layers.18.mlp.experts.74.up_proj", "model.layers.18.mlp.experts.75.up_proj", "model.layers.18.mlp.experts.76.up_proj", "model.layers.18.mlp.experts.77.up_proj", "model.layers.18.mlp.experts.78.up_proj", "model.layers.18.mlp.experts.79.up_proj", "model.layers.18.mlp.experts.80.up_proj", "model.layers.18.mlp.experts.81.up_proj", "model.layers.18.mlp.experts.82.up_proj", "model.layers.18.mlp.experts.83.up_proj", "model.layers.18.mlp.experts.84.up_proj", "model.layers.18.mlp.experts.85.up_proj", "model.layers.18.mlp.experts.86.up_proj", "model.layers.18.mlp.experts.87.up_proj", "model.layers.18.mlp.experts.88.up_proj", "model.layers.18.mlp.experts.89.up_proj", "model.layers.18.mlp.experts.90.up_proj", "model.layers.18.mlp.experts.91.up_proj", "model.layers.18.mlp.experts.92.up_proj", "model.layers.18.mlp.experts.93.up_proj", "model.layers.18.mlp.experts.94.up_proj", "model.layers.18.mlp.experts.95.up_proj", "model.layers.18.mlp.experts.96.up_proj", "model.layers.18.mlp.experts.97.up_proj", "model.layers.18.mlp.experts.98.up_proj", "model.layers.18.mlp.experts.99.up_proj", "model.layers.18.mlp.experts.100.up_proj", "model.layers.18.mlp.experts.101.up_proj", "model.layers.18.mlp.experts.102.up_proj", "model.layers.18.mlp.experts.103.up_proj", "model.layers.18.mlp.experts.104.up_proj", "model.layers.18.mlp.experts.105.up_proj", "model.layers.18.mlp.experts.106.up_proj", "model.layers.18.mlp.experts.107.up_proj", "model.layers.18.mlp.experts.108.up_proj", "model.layers.18.mlp.experts.109.up_proj", "model.layers.18.mlp.experts.110.up_proj", "model.layers.18.mlp.experts.111.up_proj", "model.layers.18.mlp.experts.112.up_proj", "model.layers.18.mlp.experts.113.up_proj", "model.layers.18.mlp.experts.114.up_proj", "model.layers.18.mlp.experts.115.up_proj", "model.layers.18.mlp.experts.116.up_proj", "model.layers.18.mlp.experts.117.up_proj", "model.layers.18.mlp.experts.118.up_proj", "model.layers.18.mlp.experts.119.up_proj", "model.layers.18.mlp.experts.120.up_proj", "model.layers.18.mlp.experts.121.up_proj", "model.layers.18.mlp.experts.122.up_proj", "model.layers.18.mlp.experts.123.up_proj", "model.layers.18.mlp.experts.124.up_proj", "model.layers.18.mlp.experts.125.up_proj", "model.layers.18.mlp.experts.126.up_proj", "model.layers.18.mlp.experts.127.up_proj", "model.layers.18.mlp.experts.128.up_proj", "model.layers.18.mlp.experts.129.up_proj", "model.layers.18.mlp.experts.130.up_proj", "model.layers.18.mlp.experts.131.up_proj", "model.layers.18.mlp.experts.132.up_proj", "model.layers.18.mlp.experts.133.up_proj", "model.layers.18.mlp.experts.134.up_proj", "model.layers.18.mlp.experts.135.up_proj", "model.layers.18.mlp.experts.136.up_proj", "model.layers.18.mlp.experts.137.up_proj", "model.layers.18.mlp.experts.138.up_proj", "model.layers.18.mlp.experts.139.up_proj", "model.layers.18.mlp.experts.140.up_proj", "model.layers.18.mlp.experts.141.up_proj", "model.layers.18.mlp.experts.142.up_proj", "model.layers.18.mlp.experts.143.up_proj", "model.layers.18.mlp.experts.144.up_proj", "model.layers.18.mlp.experts.145.up_proj", "model.layers.18.mlp.experts.146.up_proj", "model.layers.18.mlp.experts.147.up_proj", "model.layers.18.mlp.experts.148.up_proj", "model.layers.18.mlp.experts.149.up_proj", "model.layers.18.mlp.experts.150.up_proj", "model.layers.18.mlp.experts.151.up_proj", "model.layers.18.mlp.experts.152.up_proj", "model.layers.18.mlp.experts.153.up_proj", "model.layers.18.mlp.experts.154.up_proj", "model.layers.18.mlp.experts.155.up_proj", "model.layers.18.mlp.experts.156.up_proj", "model.layers.18.mlp.experts.157.up_proj", "model.layers.18.mlp.experts.158.up_proj", "model.layers.18.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.4763584476895587e-05, "dbits": 5033164800 } ] }, { "idx": 110, "layers": [ "model.layers.18.mlp.experts.0.down_proj", "model.layers.18.mlp.experts.1.down_proj", "model.layers.18.mlp.experts.2.down_proj", "model.layers.18.mlp.experts.3.down_proj", "model.layers.18.mlp.experts.4.down_proj", "model.layers.18.mlp.experts.5.down_proj", "model.layers.18.mlp.experts.6.down_proj", "model.layers.18.mlp.experts.7.down_proj", "model.layers.18.mlp.experts.8.down_proj", "model.layers.18.mlp.experts.9.down_proj", "model.layers.18.mlp.experts.10.down_proj", "model.layers.18.mlp.experts.11.down_proj", "model.layers.18.mlp.experts.12.down_proj", "model.layers.18.mlp.experts.13.down_proj", "model.layers.18.mlp.experts.14.down_proj", "model.layers.18.mlp.experts.15.down_proj", "model.layers.18.mlp.experts.16.down_proj", "model.layers.18.mlp.experts.17.down_proj", "model.layers.18.mlp.experts.18.down_proj", "model.layers.18.mlp.experts.19.down_proj", "model.layers.18.mlp.experts.20.down_proj", "model.layers.18.mlp.experts.21.down_proj", "model.layers.18.mlp.experts.22.down_proj", "model.layers.18.mlp.experts.23.down_proj", "model.layers.18.mlp.experts.24.down_proj", "model.layers.18.mlp.experts.25.down_proj", "model.layers.18.mlp.experts.26.down_proj", "model.layers.18.mlp.experts.27.down_proj", "model.layers.18.mlp.experts.28.down_proj", "model.layers.18.mlp.experts.29.down_proj", "model.layers.18.mlp.experts.30.down_proj", "model.layers.18.mlp.experts.31.down_proj", "model.layers.18.mlp.experts.32.down_proj", "model.layers.18.mlp.experts.33.down_proj", "model.layers.18.mlp.experts.34.down_proj", "model.layers.18.mlp.experts.35.down_proj", "model.layers.18.mlp.experts.36.down_proj", "model.layers.18.mlp.experts.37.down_proj", "model.layers.18.mlp.experts.38.down_proj", "model.layers.18.mlp.experts.39.down_proj", "model.layers.18.mlp.experts.40.down_proj", "model.layers.18.mlp.experts.41.down_proj", "model.layers.18.mlp.experts.42.down_proj", "model.layers.18.mlp.experts.43.down_proj", "model.layers.18.mlp.experts.44.down_proj", "model.layers.18.mlp.experts.45.down_proj", "model.layers.18.mlp.experts.46.down_proj", "model.layers.18.mlp.experts.47.down_proj", "model.layers.18.mlp.experts.48.down_proj", "model.layers.18.mlp.experts.49.down_proj", "model.layers.18.mlp.experts.50.down_proj", "model.layers.18.mlp.experts.51.down_proj", "model.layers.18.mlp.experts.52.down_proj", "model.layers.18.mlp.experts.53.down_proj", "model.layers.18.mlp.experts.54.down_proj", "model.layers.18.mlp.experts.55.down_proj", "model.layers.18.mlp.experts.56.down_proj", "model.layers.18.mlp.experts.57.down_proj", "model.layers.18.mlp.experts.58.down_proj", "model.layers.18.mlp.experts.59.down_proj", "model.layers.18.mlp.experts.60.down_proj", "model.layers.18.mlp.experts.61.down_proj", "model.layers.18.mlp.experts.62.down_proj", "model.layers.18.mlp.experts.63.down_proj", "model.layers.18.mlp.experts.64.down_proj", "model.layers.18.mlp.experts.65.down_proj", "model.layers.18.mlp.experts.66.down_proj", "model.layers.18.mlp.experts.67.down_proj", "model.layers.18.mlp.experts.68.down_proj", "model.layers.18.mlp.experts.69.down_proj", "model.layers.18.mlp.experts.70.down_proj", "model.layers.18.mlp.experts.71.down_proj", "model.layers.18.mlp.experts.72.down_proj", "model.layers.18.mlp.experts.73.down_proj", "model.layers.18.mlp.experts.74.down_proj", "model.layers.18.mlp.experts.75.down_proj", "model.layers.18.mlp.experts.76.down_proj", "model.layers.18.mlp.experts.77.down_proj", "model.layers.18.mlp.experts.78.down_proj", "model.layers.18.mlp.experts.79.down_proj", "model.layers.18.mlp.experts.80.down_proj", "model.layers.18.mlp.experts.81.down_proj", "model.layers.18.mlp.experts.82.down_proj", "model.layers.18.mlp.experts.83.down_proj", "model.layers.18.mlp.experts.84.down_proj", "model.layers.18.mlp.experts.85.down_proj", "model.layers.18.mlp.experts.86.down_proj", "model.layers.18.mlp.experts.87.down_proj", "model.layers.18.mlp.experts.88.down_proj", "model.layers.18.mlp.experts.89.down_proj", "model.layers.18.mlp.experts.90.down_proj", "model.layers.18.mlp.experts.91.down_proj", "model.layers.18.mlp.experts.92.down_proj", "model.layers.18.mlp.experts.93.down_proj", "model.layers.18.mlp.experts.94.down_proj", "model.layers.18.mlp.experts.95.down_proj", "model.layers.18.mlp.experts.96.down_proj", "model.layers.18.mlp.experts.97.down_proj", "model.layers.18.mlp.experts.98.down_proj", "model.layers.18.mlp.experts.99.down_proj", "model.layers.18.mlp.experts.100.down_proj", "model.layers.18.mlp.experts.101.down_proj", "model.layers.18.mlp.experts.102.down_proj", "model.layers.18.mlp.experts.103.down_proj", "model.layers.18.mlp.experts.104.down_proj", "model.layers.18.mlp.experts.105.down_proj", "model.layers.18.mlp.experts.106.down_proj", "model.layers.18.mlp.experts.107.down_proj", "model.layers.18.mlp.experts.108.down_proj", "model.layers.18.mlp.experts.109.down_proj", "model.layers.18.mlp.experts.110.down_proj", "model.layers.18.mlp.experts.111.down_proj", "model.layers.18.mlp.experts.112.down_proj", "model.layers.18.mlp.experts.113.down_proj", "model.layers.18.mlp.experts.114.down_proj", "model.layers.18.mlp.experts.115.down_proj", "model.layers.18.mlp.experts.116.down_proj", "model.layers.18.mlp.experts.117.down_proj", "model.layers.18.mlp.experts.118.down_proj", "model.layers.18.mlp.experts.119.down_proj", "model.layers.18.mlp.experts.120.down_proj", "model.layers.18.mlp.experts.121.down_proj", "model.layers.18.mlp.experts.122.down_proj", "model.layers.18.mlp.experts.123.down_proj", "model.layers.18.mlp.experts.124.down_proj", "model.layers.18.mlp.experts.125.down_proj", "model.layers.18.mlp.experts.126.down_proj", "model.layers.18.mlp.experts.127.down_proj", "model.layers.18.mlp.experts.128.down_proj", "model.layers.18.mlp.experts.129.down_proj", "model.layers.18.mlp.experts.130.down_proj", "model.layers.18.mlp.experts.131.down_proj", "model.layers.18.mlp.experts.132.down_proj", "model.layers.18.mlp.experts.133.down_proj", "model.layers.18.mlp.experts.134.down_proj", "model.layers.18.mlp.experts.135.down_proj", "model.layers.18.mlp.experts.136.down_proj", "model.layers.18.mlp.experts.137.down_proj", "model.layers.18.mlp.experts.138.down_proj", "model.layers.18.mlp.experts.139.down_proj", "model.layers.18.mlp.experts.140.down_proj", "model.layers.18.mlp.experts.141.down_proj", "model.layers.18.mlp.experts.142.down_proj", "model.layers.18.mlp.experts.143.down_proj", "model.layers.18.mlp.experts.144.down_proj", "model.layers.18.mlp.experts.145.down_proj", "model.layers.18.mlp.experts.146.down_proj", "model.layers.18.mlp.experts.147.down_proj", "model.layers.18.mlp.experts.148.down_proj", "model.layers.18.mlp.experts.149.down_proj", "model.layers.18.mlp.experts.150.down_proj", "model.layers.18.mlp.experts.151.down_proj", "model.layers.18.mlp.experts.152.down_proj", "model.layers.18.mlp.experts.153.down_proj", "model.layers.18.mlp.experts.154.down_proj", "model.layers.18.mlp.experts.155.down_proj", "model.layers.18.mlp.experts.156.down_proj", "model.layers.18.mlp.experts.157.down_proj", "model.layers.18.mlp.experts.158.down_proj", "model.layers.18.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.9418803751468615e-05, "dbits": 2516582400 } ] }, { "idx": 111, "layers": [ "model.layers.19.self_attn.q_proj" ], "candidates": [ { "dkld": 1.6121074440889017e-05, "dbits": 125829120 } ] }, { "idx": 112, "layers": [ "model.layers.19.self_attn.k_proj", "model.layers.19.self_attn.v_proj" ], "candidates": [ { "dkld": -8.518277900293491e-06, "dbits": 20971520 } ] }, { "idx": 113, "layers": [ "model.layers.19.self_attn.o_proj" ], "candidates": [ { "dkld": -1.1314012226648629e-05, "dbits": 125829120 } ] }, { "idx": 114, "layers": [ "model.layers.19.mlp.shared_experts.gate_proj", "model.layers.19.mlp.shared_experts.up_proj", "model.layers.19.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -8.117250399664088e-06, "dbits": 47185920 } ] }, { "idx": 115, "layers": [ "model.layers.19.mlp.experts.0.gate_proj", "model.layers.19.mlp.experts.1.gate_proj", "model.layers.19.mlp.experts.2.gate_proj", "model.layers.19.mlp.experts.3.gate_proj", "model.layers.19.mlp.experts.4.gate_proj", "model.layers.19.mlp.experts.5.gate_proj", "model.layers.19.mlp.experts.6.gate_proj", "model.layers.19.mlp.experts.7.gate_proj", "model.layers.19.mlp.experts.8.gate_proj", "model.layers.19.mlp.experts.9.gate_proj", "model.layers.19.mlp.experts.10.gate_proj", "model.layers.19.mlp.experts.11.gate_proj", "model.layers.19.mlp.experts.12.gate_proj", "model.layers.19.mlp.experts.13.gate_proj", "model.layers.19.mlp.experts.14.gate_proj", "model.layers.19.mlp.experts.15.gate_proj", "model.layers.19.mlp.experts.16.gate_proj", "model.layers.19.mlp.experts.17.gate_proj", "model.layers.19.mlp.experts.18.gate_proj", "model.layers.19.mlp.experts.19.gate_proj", "model.layers.19.mlp.experts.20.gate_proj", "model.layers.19.mlp.experts.21.gate_proj", "model.layers.19.mlp.experts.22.gate_proj", "model.layers.19.mlp.experts.23.gate_proj", "model.layers.19.mlp.experts.24.gate_proj", "model.layers.19.mlp.experts.25.gate_proj", "model.layers.19.mlp.experts.26.gate_proj", "model.layers.19.mlp.experts.27.gate_proj", "model.layers.19.mlp.experts.28.gate_proj", "model.layers.19.mlp.experts.29.gate_proj", "model.layers.19.mlp.experts.30.gate_proj", "model.layers.19.mlp.experts.31.gate_proj", "model.layers.19.mlp.experts.32.gate_proj", "model.layers.19.mlp.experts.33.gate_proj", "model.layers.19.mlp.experts.34.gate_proj", "model.layers.19.mlp.experts.35.gate_proj", "model.layers.19.mlp.experts.36.gate_proj", "model.layers.19.mlp.experts.37.gate_proj", "model.layers.19.mlp.experts.38.gate_proj", "model.layers.19.mlp.experts.39.gate_proj", "model.layers.19.mlp.experts.40.gate_proj", "model.layers.19.mlp.experts.41.gate_proj", "model.layers.19.mlp.experts.42.gate_proj", "model.layers.19.mlp.experts.43.gate_proj", "model.layers.19.mlp.experts.44.gate_proj", "model.layers.19.mlp.experts.45.gate_proj", "model.layers.19.mlp.experts.46.gate_proj", "model.layers.19.mlp.experts.47.gate_proj", "model.layers.19.mlp.experts.48.gate_proj", "model.layers.19.mlp.experts.49.gate_proj", "model.layers.19.mlp.experts.50.gate_proj", "model.layers.19.mlp.experts.51.gate_proj", "model.layers.19.mlp.experts.52.gate_proj", "model.layers.19.mlp.experts.53.gate_proj", "model.layers.19.mlp.experts.54.gate_proj", "model.layers.19.mlp.experts.55.gate_proj", "model.layers.19.mlp.experts.56.gate_proj", "model.layers.19.mlp.experts.57.gate_proj", "model.layers.19.mlp.experts.58.gate_proj", "model.layers.19.mlp.experts.59.gate_proj", "model.layers.19.mlp.experts.60.gate_proj", "model.layers.19.mlp.experts.61.gate_proj", "model.layers.19.mlp.experts.62.gate_proj", "model.layers.19.mlp.experts.63.gate_proj", "model.layers.19.mlp.experts.64.gate_proj", "model.layers.19.mlp.experts.65.gate_proj", "model.layers.19.mlp.experts.66.gate_proj", "model.layers.19.mlp.experts.67.gate_proj", "model.layers.19.mlp.experts.68.gate_proj", "model.layers.19.mlp.experts.69.gate_proj", "model.layers.19.mlp.experts.70.gate_proj", "model.layers.19.mlp.experts.71.gate_proj", "model.layers.19.mlp.experts.72.gate_proj", "model.layers.19.mlp.experts.73.gate_proj", "model.layers.19.mlp.experts.74.gate_proj", "model.layers.19.mlp.experts.75.gate_proj", "model.layers.19.mlp.experts.76.gate_proj", "model.layers.19.mlp.experts.77.gate_proj", "model.layers.19.mlp.experts.78.gate_proj", "model.layers.19.mlp.experts.79.gate_proj", "model.layers.19.mlp.experts.80.gate_proj", "model.layers.19.mlp.experts.81.gate_proj", "model.layers.19.mlp.experts.82.gate_proj", "model.layers.19.mlp.experts.83.gate_proj", "model.layers.19.mlp.experts.84.gate_proj", "model.layers.19.mlp.experts.85.gate_proj", "model.layers.19.mlp.experts.86.gate_proj", "model.layers.19.mlp.experts.87.gate_proj", "model.layers.19.mlp.experts.88.gate_proj", "model.layers.19.mlp.experts.89.gate_proj", "model.layers.19.mlp.experts.90.gate_proj", "model.layers.19.mlp.experts.91.gate_proj", "model.layers.19.mlp.experts.92.gate_proj", "model.layers.19.mlp.experts.93.gate_proj", "model.layers.19.mlp.experts.94.gate_proj", "model.layers.19.mlp.experts.95.gate_proj", "model.layers.19.mlp.experts.96.gate_proj", "model.layers.19.mlp.experts.97.gate_proj", "model.layers.19.mlp.experts.98.gate_proj", "model.layers.19.mlp.experts.99.gate_proj", "model.layers.19.mlp.experts.100.gate_proj", "model.layers.19.mlp.experts.101.gate_proj", "model.layers.19.mlp.experts.102.gate_proj", "model.layers.19.mlp.experts.103.gate_proj", "model.layers.19.mlp.experts.104.gate_proj", "model.layers.19.mlp.experts.105.gate_proj", "model.layers.19.mlp.experts.106.gate_proj", "model.layers.19.mlp.experts.107.gate_proj", "model.layers.19.mlp.experts.108.gate_proj", "model.layers.19.mlp.experts.109.gate_proj", "model.layers.19.mlp.experts.110.gate_proj", "model.layers.19.mlp.experts.111.gate_proj", "model.layers.19.mlp.experts.112.gate_proj", "model.layers.19.mlp.experts.113.gate_proj", "model.layers.19.mlp.experts.114.gate_proj", "model.layers.19.mlp.experts.115.gate_proj", "model.layers.19.mlp.experts.116.gate_proj", "model.layers.19.mlp.experts.117.gate_proj", "model.layers.19.mlp.experts.118.gate_proj", "model.layers.19.mlp.experts.119.gate_proj", "model.layers.19.mlp.experts.120.gate_proj", "model.layers.19.mlp.experts.121.gate_proj", "model.layers.19.mlp.experts.122.gate_proj", "model.layers.19.mlp.experts.123.gate_proj", "model.layers.19.mlp.experts.124.gate_proj", "model.layers.19.mlp.experts.125.gate_proj", "model.layers.19.mlp.experts.126.gate_proj", "model.layers.19.mlp.experts.127.gate_proj", "model.layers.19.mlp.experts.128.gate_proj", "model.layers.19.mlp.experts.129.gate_proj", "model.layers.19.mlp.experts.130.gate_proj", "model.layers.19.mlp.experts.131.gate_proj", "model.layers.19.mlp.experts.132.gate_proj", "model.layers.19.mlp.experts.133.gate_proj", "model.layers.19.mlp.experts.134.gate_proj", "model.layers.19.mlp.experts.135.gate_proj", "model.layers.19.mlp.experts.136.gate_proj", "model.layers.19.mlp.experts.137.gate_proj", "model.layers.19.mlp.experts.138.gate_proj", "model.layers.19.mlp.experts.139.gate_proj", "model.layers.19.mlp.experts.140.gate_proj", "model.layers.19.mlp.experts.141.gate_proj", "model.layers.19.mlp.experts.142.gate_proj", "model.layers.19.mlp.experts.143.gate_proj", "model.layers.19.mlp.experts.144.gate_proj", "model.layers.19.mlp.experts.145.gate_proj", "model.layers.19.mlp.experts.146.gate_proj", "model.layers.19.mlp.experts.147.gate_proj", "model.layers.19.mlp.experts.148.gate_proj", "model.layers.19.mlp.experts.149.gate_proj", "model.layers.19.mlp.experts.150.gate_proj", "model.layers.19.mlp.experts.151.gate_proj", "model.layers.19.mlp.experts.152.gate_proj", "model.layers.19.mlp.experts.153.gate_proj", "model.layers.19.mlp.experts.154.gate_proj", "model.layers.19.mlp.experts.155.gate_proj", "model.layers.19.mlp.experts.156.gate_proj", "model.layers.19.mlp.experts.157.gate_proj", "model.layers.19.mlp.experts.158.gate_proj", "model.layers.19.mlp.experts.159.gate_proj", "model.layers.19.mlp.experts.0.up_proj", "model.layers.19.mlp.experts.1.up_proj", "model.layers.19.mlp.experts.2.up_proj", "model.layers.19.mlp.experts.3.up_proj", "model.layers.19.mlp.experts.4.up_proj", "model.layers.19.mlp.experts.5.up_proj", "model.layers.19.mlp.experts.6.up_proj", "model.layers.19.mlp.experts.7.up_proj", "model.layers.19.mlp.experts.8.up_proj", "model.layers.19.mlp.experts.9.up_proj", "model.layers.19.mlp.experts.10.up_proj", "model.layers.19.mlp.experts.11.up_proj", "model.layers.19.mlp.experts.12.up_proj", "model.layers.19.mlp.experts.13.up_proj", "model.layers.19.mlp.experts.14.up_proj", "model.layers.19.mlp.experts.15.up_proj", "model.layers.19.mlp.experts.16.up_proj", "model.layers.19.mlp.experts.17.up_proj", "model.layers.19.mlp.experts.18.up_proj", "model.layers.19.mlp.experts.19.up_proj", "model.layers.19.mlp.experts.20.up_proj", "model.layers.19.mlp.experts.21.up_proj", "model.layers.19.mlp.experts.22.up_proj", "model.layers.19.mlp.experts.23.up_proj", "model.layers.19.mlp.experts.24.up_proj", "model.layers.19.mlp.experts.25.up_proj", "model.layers.19.mlp.experts.26.up_proj", "model.layers.19.mlp.experts.27.up_proj", "model.layers.19.mlp.experts.28.up_proj", "model.layers.19.mlp.experts.29.up_proj", "model.layers.19.mlp.experts.30.up_proj", "model.layers.19.mlp.experts.31.up_proj", "model.layers.19.mlp.experts.32.up_proj", "model.layers.19.mlp.experts.33.up_proj", "model.layers.19.mlp.experts.34.up_proj", "model.layers.19.mlp.experts.35.up_proj", "model.layers.19.mlp.experts.36.up_proj", "model.layers.19.mlp.experts.37.up_proj", "model.layers.19.mlp.experts.38.up_proj", "model.layers.19.mlp.experts.39.up_proj", "model.layers.19.mlp.experts.40.up_proj", "model.layers.19.mlp.experts.41.up_proj", "model.layers.19.mlp.experts.42.up_proj", "model.layers.19.mlp.experts.43.up_proj", "model.layers.19.mlp.experts.44.up_proj", "model.layers.19.mlp.experts.45.up_proj", "model.layers.19.mlp.experts.46.up_proj", "model.layers.19.mlp.experts.47.up_proj", "model.layers.19.mlp.experts.48.up_proj", "model.layers.19.mlp.experts.49.up_proj", "model.layers.19.mlp.experts.50.up_proj", "model.layers.19.mlp.experts.51.up_proj", "model.layers.19.mlp.experts.52.up_proj", "model.layers.19.mlp.experts.53.up_proj", "model.layers.19.mlp.experts.54.up_proj", "model.layers.19.mlp.experts.55.up_proj", "model.layers.19.mlp.experts.56.up_proj", "model.layers.19.mlp.experts.57.up_proj", "model.layers.19.mlp.experts.58.up_proj", "model.layers.19.mlp.experts.59.up_proj", "model.layers.19.mlp.experts.60.up_proj", "model.layers.19.mlp.experts.61.up_proj", "model.layers.19.mlp.experts.62.up_proj", "model.layers.19.mlp.experts.63.up_proj", "model.layers.19.mlp.experts.64.up_proj", "model.layers.19.mlp.experts.65.up_proj", "model.layers.19.mlp.experts.66.up_proj", "model.layers.19.mlp.experts.67.up_proj", "model.layers.19.mlp.experts.68.up_proj", "model.layers.19.mlp.experts.69.up_proj", "model.layers.19.mlp.experts.70.up_proj", "model.layers.19.mlp.experts.71.up_proj", "model.layers.19.mlp.experts.72.up_proj", "model.layers.19.mlp.experts.73.up_proj", "model.layers.19.mlp.experts.74.up_proj", "model.layers.19.mlp.experts.75.up_proj", "model.layers.19.mlp.experts.76.up_proj", "model.layers.19.mlp.experts.77.up_proj", "model.layers.19.mlp.experts.78.up_proj", "model.layers.19.mlp.experts.79.up_proj", "model.layers.19.mlp.experts.80.up_proj", "model.layers.19.mlp.experts.81.up_proj", "model.layers.19.mlp.experts.82.up_proj", "model.layers.19.mlp.experts.83.up_proj", "model.layers.19.mlp.experts.84.up_proj", "model.layers.19.mlp.experts.85.up_proj", "model.layers.19.mlp.experts.86.up_proj", "model.layers.19.mlp.experts.87.up_proj", "model.layers.19.mlp.experts.88.up_proj", "model.layers.19.mlp.experts.89.up_proj", "model.layers.19.mlp.experts.90.up_proj", "model.layers.19.mlp.experts.91.up_proj", "model.layers.19.mlp.experts.92.up_proj", "model.layers.19.mlp.experts.93.up_proj", "model.layers.19.mlp.experts.94.up_proj", "model.layers.19.mlp.experts.95.up_proj", "model.layers.19.mlp.experts.96.up_proj", "model.layers.19.mlp.experts.97.up_proj", "model.layers.19.mlp.experts.98.up_proj", "model.layers.19.mlp.experts.99.up_proj", "model.layers.19.mlp.experts.100.up_proj", "model.layers.19.mlp.experts.101.up_proj", "model.layers.19.mlp.experts.102.up_proj", "model.layers.19.mlp.experts.103.up_proj", "model.layers.19.mlp.experts.104.up_proj", "model.layers.19.mlp.experts.105.up_proj", "model.layers.19.mlp.experts.106.up_proj", "model.layers.19.mlp.experts.107.up_proj", "model.layers.19.mlp.experts.108.up_proj", "model.layers.19.mlp.experts.109.up_proj", "model.layers.19.mlp.experts.110.up_proj", "model.layers.19.mlp.experts.111.up_proj", "model.layers.19.mlp.experts.112.up_proj", "model.layers.19.mlp.experts.113.up_proj", "model.layers.19.mlp.experts.114.up_proj", "model.layers.19.mlp.experts.115.up_proj", "model.layers.19.mlp.experts.116.up_proj", "model.layers.19.mlp.experts.117.up_proj", "model.layers.19.mlp.experts.118.up_proj", "model.layers.19.mlp.experts.119.up_proj", "model.layers.19.mlp.experts.120.up_proj", "model.layers.19.mlp.experts.121.up_proj", "model.layers.19.mlp.experts.122.up_proj", "model.layers.19.mlp.experts.123.up_proj", "model.layers.19.mlp.experts.124.up_proj", "model.layers.19.mlp.experts.125.up_proj", "model.layers.19.mlp.experts.126.up_proj", "model.layers.19.mlp.experts.127.up_proj", "model.layers.19.mlp.experts.128.up_proj", "model.layers.19.mlp.experts.129.up_proj", "model.layers.19.mlp.experts.130.up_proj", "model.layers.19.mlp.experts.131.up_proj", "model.layers.19.mlp.experts.132.up_proj", "model.layers.19.mlp.experts.133.up_proj", "model.layers.19.mlp.experts.134.up_proj", "model.layers.19.mlp.experts.135.up_proj", "model.layers.19.mlp.experts.136.up_proj", "model.layers.19.mlp.experts.137.up_proj", "model.layers.19.mlp.experts.138.up_proj", "model.layers.19.mlp.experts.139.up_proj", "model.layers.19.mlp.experts.140.up_proj", "model.layers.19.mlp.experts.141.up_proj", "model.layers.19.mlp.experts.142.up_proj", "model.layers.19.mlp.experts.143.up_proj", "model.layers.19.mlp.experts.144.up_proj", "model.layers.19.mlp.experts.145.up_proj", "model.layers.19.mlp.experts.146.up_proj", "model.layers.19.mlp.experts.147.up_proj", "model.layers.19.mlp.experts.148.up_proj", "model.layers.19.mlp.experts.149.up_proj", "model.layers.19.mlp.experts.150.up_proj", "model.layers.19.mlp.experts.151.up_proj", "model.layers.19.mlp.experts.152.up_proj", "model.layers.19.mlp.experts.153.up_proj", "model.layers.19.mlp.experts.154.up_proj", "model.layers.19.mlp.experts.155.up_proj", "model.layers.19.mlp.experts.156.up_proj", "model.layers.19.mlp.experts.157.up_proj", "model.layers.19.mlp.experts.158.up_proj", "model.layers.19.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.51141565665594e-06, "dbits": 5033164800 } ] }, { "idx": 116, "layers": [ "model.layers.19.mlp.experts.0.down_proj", "model.layers.19.mlp.experts.1.down_proj", "model.layers.19.mlp.experts.2.down_proj", "model.layers.19.mlp.experts.3.down_proj", "model.layers.19.mlp.experts.4.down_proj", "model.layers.19.mlp.experts.5.down_proj", "model.layers.19.mlp.experts.6.down_proj", "model.layers.19.mlp.experts.7.down_proj", "model.layers.19.mlp.experts.8.down_proj", "model.layers.19.mlp.experts.9.down_proj", "model.layers.19.mlp.experts.10.down_proj", "model.layers.19.mlp.experts.11.down_proj", "model.layers.19.mlp.experts.12.down_proj", "model.layers.19.mlp.experts.13.down_proj", "model.layers.19.mlp.experts.14.down_proj", "model.layers.19.mlp.experts.15.down_proj", "model.layers.19.mlp.experts.16.down_proj", "model.layers.19.mlp.experts.17.down_proj", "model.layers.19.mlp.experts.18.down_proj", "model.layers.19.mlp.experts.19.down_proj", "model.layers.19.mlp.experts.20.down_proj", "model.layers.19.mlp.experts.21.down_proj", "model.layers.19.mlp.experts.22.down_proj", "model.layers.19.mlp.experts.23.down_proj", "model.layers.19.mlp.experts.24.down_proj", "model.layers.19.mlp.experts.25.down_proj", "model.layers.19.mlp.experts.26.down_proj", "model.layers.19.mlp.experts.27.down_proj", "model.layers.19.mlp.experts.28.down_proj", "model.layers.19.mlp.experts.29.down_proj", "model.layers.19.mlp.experts.30.down_proj", "model.layers.19.mlp.experts.31.down_proj", "model.layers.19.mlp.experts.32.down_proj", "model.layers.19.mlp.experts.33.down_proj", "model.layers.19.mlp.experts.34.down_proj", "model.layers.19.mlp.experts.35.down_proj", "model.layers.19.mlp.experts.36.down_proj", "model.layers.19.mlp.experts.37.down_proj", "model.layers.19.mlp.experts.38.down_proj", "model.layers.19.mlp.experts.39.down_proj", "model.layers.19.mlp.experts.40.down_proj", "model.layers.19.mlp.experts.41.down_proj", "model.layers.19.mlp.experts.42.down_proj", "model.layers.19.mlp.experts.43.down_proj", "model.layers.19.mlp.experts.44.down_proj", "model.layers.19.mlp.experts.45.down_proj", "model.layers.19.mlp.experts.46.down_proj", "model.layers.19.mlp.experts.47.down_proj", "model.layers.19.mlp.experts.48.down_proj", "model.layers.19.mlp.experts.49.down_proj", "model.layers.19.mlp.experts.50.down_proj", "model.layers.19.mlp.experts.51.down_proj", "model.layers.19.mlp.experts.52.down_proj", "model.layers.19.mlp.experts.53.down_proj", "model.layers.19.mlp.experts.54.down_proj", "model.layers.19.mlp.experts.55.down_proj", "model.layers.19.mlp.experts.56.down_proj", "model.layers.19.mlp.experts.57.down_proj", "model.layers.19.mlp.experts.58.down_proj", "model.layers.19.mlp.experts.59.down_proj", "model.layers.19.mlp.experts.60.down_proj", "model.layers.19.mlp.experts.61.down_proj", "model.layers.19.mlp.experts.62.down_proj", "model.layers.19.mlp.experts.63.down_proj", "model.layers.19.mlp.experts.64.down_proj", "model.layers.19.mlp.experts.65.down_proj", "model.layers.19.mlp.experts.66.down_proj", "model.layers.19.mlp.experts.67.down_proj", "model.layers.19.mlp.experts.68.down_proj", "model.layers.19.mlp.experts.69.down_proj", "model.layers.19.mlp.experts.70.down_proj", "model.layers.19.mlp.experts.71.down_proj", "model.layers.19.mlp.experts.72.down_proj", "model.layers.19.mlp.experts.73.down_proj", "model.layers.19.mlp.experts.74.down_proj", "model.layers.19.mlp.experts.75.down_proj", "model.layers.19.mlp.experts.76.down_proj", "model.layers.19.mlp.experts.77.down_proj", "model.layers.19.mlp.experts.78.down_proj", "model.layers.19.mlp.experts.79.down_proj", "model.layers.19.mlp.experts.80.down_proj", "model.layers.19.mlp.experts.81.down_proj", "model.layers.19.mlp.experts.82.down_proj", "model.layers.19.mlp.experts.83.down_proj", "model.layers.19.mlp.experts.84.down_proj", "model.layers.19.mlp.experts.85.down_proj", "model.layers.19.mlp.experts.86.down_proj", "model.layers.19.mlp.experts.87.down_proj", "model.layers.19.mlp.experts.88.down_proj", "model.layers.19.mlp.experts.89.down_proj", "model.layers.19.mlp.experts.90.down_proj", "model.layers.19.mlp.experts.91.down_proj", "model.layers.19.mlp.experts.92.down_proj", "model.layers.19.mlp.experts.93.down_proj", "model.layers.19.mlp.experts.94.down_proj", "model.layers.19.mlp.experts.95.down_proj", "model.layers.19.mlp.experts.96.down_proj", "model.layers.19.mlp.experts.97.down_proj", "model.layers.19.mlp.experts.98.down_proj", "model.layers.19.mlp.experts.99.down_proj", "model.layers.19.mlp.experts.100.down_proj", "model.layers.19.mlp.experts.101.down_proj", "model.layers.19.mlp.experts.102.down_proj", "model.layers.19.mlp.experts.103.down_proj", "model.layers.19.mlp.experts.104.down_proj", "model.layers.19.mlp.experts.105.down_proj", "model.layers.19.mlp.experts.106.down_proj", "model.layers.19.mlp.experts.107.down_proj", "model.layers.19.mlp.experts.108.down_proj", "model.layers.19.mlp.experts.109.down_proj", "model.layers.19.mlp.experts.110.down_proj", "model.layers.19.mlp.experts.111.down_proj", "model.layers.19.mlp.experts.112.down_proj", "model.layers.19.mlp.experts.113.down_proj", "model.layers.19.mlp.experts.114.down_proj", "model.layers.19.mlp.experts.115.down_proj", "model.layers.19.mlp.experts.116.down_proj", "model.layers.19.mlp.experts.117.down_proj", "model.layers.19.mlp.experts.118.down_proj", "model.layers.19.mlp.experts.119.down_proj", "model.layers.19.mlp.experts.120.down_proj", "model.layers.19.mlp.experts.121.down_proj", "model.layers.19.mlp.experts.122.down_proj", "model.layers.19.mlp.experts.123.down_proj", "model.layers.19.mlp.experts.124.down_proj", "model.layers.19.mlp.experts.125.down_proj", "model.layers.19.mlp.experts.126.down_proj", "model.layers.19.mlp.experts.127.down_proj", "model.layers.19.mlp.experts.128.down_proj", "model.layers.19.mlp.experts.129.down_proj", "model.layers.19.mlp.experts.130.down_proj", "model.layers.19.mlp.experts.131.down_proj", "model.layers.19.mlp.experts.132.down_proj", "model.layers.19.mlp.experts.133.down_proj", "model.layers.19.mlp.experts.134.down_proj", "model.layers.19.mlp.experts.135.down_proj", "model.layers.19.mlp.experts.136.down_proj", "model.layers.19.mlp.experts.137.down_proj", "model.layers.19.mlp.experts.138.down_proj", "model.layers.19.mlp.experts.139.down_proj", "model.layers.19.mlp.experts.140.down_proj", "model.layers.19.mlp.experts.141.down_proj", "model.layers.19.mlp.experts.142.down_proj", "model.layers.19.mlp.experts.143.down_proj", "model.layers.19.mlp.experts.144.down_proj", "model.layers.19.mlp.experts.145.down_proj", "model.layers.19.mlp.experts.146.down_proj", "model.layers.19.mlp.experts.147.down_proj", "model.layers.19.mlp.experts.148.down_proj", "model.layers.19.mlp.experts.149.down_proj", "model.layers.19.mlp.experts.150.down_proj", "model.layers.19.mlp.experts.151.down_proj", "model.layers.19.mlp.experts.152.down_proj", "model.layers.19.mlp.experts.153.down_proj", "model.layers.19.mlp.experts.154.down_proj", "model.layers.19.mlp.experts.155.down_proj", "model.layers.19.mlp.experts.156.down_proj", "model.layers.19.mlp.experts.157.down_proj", "model.layers.19.mlp.experts.158.down_proj", "model.layers.19.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.7987890169024446e-05, "dbits": 2516582400 } ] }, { "idx": 117, "layers": [ "model.layers.20.self_attn.q_proj" ], "candidates": [ { "dkld": -4.544854164127872e-08, "dbits": 125829120 } ] }, { "idx": 118, "layers": [ "model.layers.20.self_attn.k_proj", "model.layers.20.self_attn.v_proj" ], "candidates": [ { "dkld": -2.1031484357081375e-05, "dbits": 20971520 } ] }, { "idx": 119, "layers": [ "model.layers.20.self_attn.o_proj" ], "candidates": [ { "dkld": -5.320092895999334e-07, "dbits": 125829120 } ] }, { "idx": 120, "layers": [ "model.layers.20.mlp.shared_experts.gate_proj", "model.layers.20.mlp.shared_experts.up_proj", "model.layers.20.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.4405485126189866e-05, "dbits": 47185920 } ] }, { "idx": 121, "layers": [ "model.layers.20.mlp.experts.0.gate_proj", "model.layers.20.mlp.experts.1.gate_proj", "model.layers.20.mlp.experts.2.gate_proj", "model.layers.20.mlp.experts.3.gate_proj", "model.layers.20.mlp.experts.4.gate_proj", "model.layers.20.mlp.experts.5.gate_proj", "model.layers.20.mlp.experts.6.gate_proj", "model.layers.20.mlp.experts.7.gate_proj", "model.layers.20.mlp.experts.8.gate_proj", "model.layers.20.mlp.experts.9.gate_proj", "model.layers.20.mlp.experts.10.gate_proj", "model.layers.20.mlp.experts.11.gate_proj", "model.layers.20.mlp.experts.12.gate_proj", "model.layers.20.mlp.experts.13.gate_proj", "model.layers.20.mlp.experts.14.gate_proj", "model.layers.20.mlp.experts.15.gate_proj", "model.layers.20.mlp.experts.16.gate_proj", "model.layers.20.mlp.experts.17.gate_proj", "model.layers.20.mlp.experts.18.gate_proj", "model.layers.20.mlp.experts.19.gate_proj", "model.layers.20.mlp.experts.20.gate_proj", "model.layers.20.mlp.experts.21.gate_proj", "model.layers.20.mlp.experts.22.gate_proj", "model.layers.20.mlp.experts.23.gate_proj", "model.layers.20.mlp.experts.24.gate_proj", "model.layers.20.mlp.experts.25.gate_proj", "model.layers.20.mlp.experts.26.gate_proj", "model.layers.20.mlp.experts.27.gate_proj", "model.layers.20.mlp.experts.28.gate_proj", "model.layers.20.mlp.experts.29.gate_proj", "model.layers.20.mlp.experts.30.gate_proj", "model.layers.20.mlp.experts.31.gate_proj", "model.layers.20.mlp.experts.32.gate_proj", "model.layers.20.mlp.experts.33.gate_proj", "model.layers.20.mlp.experts.34.gate_proj", "model.layers.20.mlp.experts.35.gate_proj", "model.layers.20.mlp.experts.36.gate_proj", "model.layers.20.mlp.experts.37.gate_proj", "model.layers.20.mlp.experts.38.gate_proj", "model.layers.20.mlp.experts.39.gate_proj", "model.layers.20.mlp.experts.40.gate_proj", "model.layers.20.mlp.experts.41.gate_proj", "model.layers.20.mlp.experts.42.gate_proj", "model.layers.20.mlp.experts.43.gate_proj", "model.layers.20.mlp.experts.44.gate_proj", "model.layers.20.mlp.experts.45.gate_proj", "model.layers.20.mlp.experts.46.gate_proj", "model.layers.20.mlp.experts.47.gate_proj", "model.layers.20.mlp.experts.48.gate_proj", "model.layers.20.mlp.experts.49.gate_proj", "model.layers.20.mlp.experts.50.gate_proj", "model.layers.20.mlp.experts.51.gate_proj", "model.layers.20.mlp.experts.52.gate_proj", "model.layers.20.mlp.experts.53.gate_proj", "model.layers.20.mlp.experts.54.gate_proj", "model.layers.20.mlp.experts.55.gate_proj", "model.layers.20.mlp.experts.56.gate_proj", "model.layers.20.mlp.experts.57.gate_proj", "model.layers.20.mlp.experts.58.gate_proj", "model.layers.20.mlp.experts.59.gate_proj", "model.layers.20.mlp.experts.60.gate_proj", "model.layers.20.mlp.experts.61.gate_proj", "model.layers.20.mlp.experts.62.gate_proj", "model.layers.20.mlp.experts.63.gate_proj", "model.layers.20.mlp.experts.64.gate_proj", "model.layers.20.mlp.experts.65.gate_proj", "model.layers.20.mlp.experts.66.gate_proj", "model.layers.20.mlp.experts.67.gate_proj", "model.layers.20.mlp.experts.68.gate_proj", "model.layers.20.mlp.experts.69.gate_proj", "model.layers.20.mlp.experts.70.gate_proj", "model.layers.20.mlp.experts.71.gate_proj", "model.layers.20.mlp.experts.72.gate_proj", "model.layers.20.mlp.experts.73.gate_proj", "model.layers.20.mlp.experts.74.gate_proj", "model.layers.20.mlp.experts.75.gate_proj", "model.layers.20.mlp.experts.76.gate_proj", "model.layers.20.mlp.experts.77.gate_proj", "model.layers.20.mlp.experts.78.gate_proj", "model.layers.20.mlp.experts.79.gate_proj", "model.layers.20.mlp.experts.80.gate_proj", "model.layers.20.mlp.experts.81.gate_proj", "model.layers.20.mlp.experts.82.gate_proj", "model.layers.20.mlp.experts.83.gate_proj", "model.layers.20.mlp.experts.84.gate_proj", "model.layers.20.mlp.experts.85.gate_proj", "model.layers.20.mlp.experts.86.gate_proj", "model.layers.20.mlp.experts.87.gate_proj", "model.layers.20.mlp.experts.88.gate_proj", "model.layers.20.mlp.experts.89.gate_proj", "model.layers.20.mlp.experts.90.gate_proj", "model.layers.20.mlp.experts.91.gate_proj", "model.layers.20.mlp.experts.92.gate_proj", "model.layers.20.mlp.experts.93.gate_proj", "model.layers.20.mlp.experts.94.gate_proj", "model.layers.20.mlp.experts.95.gate_proj", "model.layers.20.mlp.experts.96.gate_proj", "model.layers.20.mlp.experts.97.gate_proj", "model.layers.20.mlp.experts.98.gate_proj", "model.layers.20.mlp.experts.99.gate_proj", "model.layers.20.mlp.experts.100.gate_proj", "model.layers.20.mlp.experts.101.gate_proj", "model.layers.20.mlp.experts.102.gate_proj", "model.layers.20.mlp.experts.103.gate_proj", "model.layers.20.mlp.experts.104.gate_proj", "model.layers.20.mlp.experts.105.gate_proj", "model.layers.20.mlp.experts.106.gate_proj", "model.layers.20.mlp.experts.107.gate_proj", "model.layers.20.mlp.experts.108.gate_proj", "model.layers.20.mlp.experts.109.gate_proj", "model.layers.20.mlp.experts.110.gate_proj", "model.layers.20.mlp.experts.111.gate_proj", "model.layers.20.mlp.experts.112.gate_proj", "model.layers.20.mlp.experts.113.gate_proj", "model.layers.20.mlp.experts.114.gate_proj", "model.layers.20.mlp.experts.115.gate_proj", "model.layers.20.mlp.experts.116.gate_proj", "model.layers.20.mlp.experts.117.gate_proj", "model.layers.20.mlp.experts.118.gate_proj", "model.layers.20.mlp.experts.119.gate_proj", "model.layers.20.mlp.experts.120.gate_proj", "model.layers.20.mlp.experts.121.gate_proj", "model.layers.20.mlp.experts.122.gate_proj", "model.layers.20.mlp.experts.123.gate_proj", "model.layers.20.mlp.experts.124.gate_proj", "model.layers.20.mlp.experts.125.gate_proj", "model.layers.20.mlp.experts.126.gate_proj", "model.layers.20.mlp.experts.127.gate_proj", "model.layers.20.mlp.experts.128.gate_proj", "model.layers.20.mlp.experts.129.gate_proj", "model.layers.20.mlp.experts.130.gate_proj", "model.layers.20.mlp.experts.131.gate_proj", "model.layers.20.mlp.experts.132.gate_proj", "model.layers.20.mlp.experts.133.gate_proj", "model.layers.20.mlp.experts.134.gate_proj", "model.layers.20.mlp.experts.135.gate_proj", "model.layers.20.mlp.experts.136.gate_proj", "model.layers.20.mlp.experts.137.gate_proj", "model.layers.20.mlp.experts.138.gate_proj", "model.layers.20.mlp.experts.139.gate_proj", "model.layers.20.mlp.experts.140.gate_proj", "model.layers.20.mlp.experts.141.gate_proj", "model.layers.20.mlp.experts.142.gate_proj", "model.layers.20.mlp.experts.143.gate_proj", "model.layers.20.mlp.experts.144.gate_proj", "model.layers.20.mlp.experts.145.gate_proj", "model.layers.20.mlp.experts.146.gate_proj", "model.layers.20.mlp.experts.147.gate_proj", "model.layers.20.mlp.experts.148.gate_proj", "model.layers.20.mlp.experts.149.gate_proj", "model.layers.20.mlp.experts.150.gate_proj", "model.layers.20.mlp.experts.151.gate_proj", "model.layers.20.mlp.experts.152.gate_proj", "model.layers.20.mlp.experts.153.gate_proj", "model.layers.20.mlp.experts.154.gate_proj", "model.layers.20.mlp.experts.155.gate_proj", "model.layers.20.mlp.experts.156.gate_proj", "model.layers.20.mlp.experts.157.gate_proj", "model.layers.20.mlp.experts.158.gate_proj", "model.layers.20.mlp.experts.159.gate_proj", "model.layers.20.mlp.experts.0.up_proj", "model.layers.20.mlp.experts.1.up_proj", "model.layers.20.mlp.experts.2.up_proj", "model.layers.20.mlp.experts.3.up_proj", "model.layers.20.mlp.experts.4.up_proj", "model.layers.20.mlp.experts.5.up_proj", "model.layers.20.mlp.experts.6.up_proj", "model.layers.20.mlp.experts.7.up_proj", "model.layers.20.mlp.experts.8.up_proj", "model.layers.20.mlp.experts.9.up_proj", "model.layers.20.mlp.experts.10.up_proj", "model.layers.20.mlp.experts.11.up_proj", "model.layers.20.mlp.experts.12.up_proj", "model.layers.20.mlp.experts.13.up_proj", "model.layers.20.mlp.experts.14.up_proj", "model.layers.20.mlp.experts.15.up_proj", "model.layers.20.mlp.experts.16.up_proj", "model.layers.20.mlp.experts.17.up_proj", "model.layers.20.mlp.experts.18.up_proj", "model.layers.20.mlp.experts.19.up_proj", "model.layers.20.mlp.experts.20.up_proj", "model.layers.20.mlp.experts.21.up_proj", "model.layers.20.mlp.experts.22.up_proj", "model.layers.20.mlp.experts.23.up_proj", "model.layers.20.mlp.experts.24.up_proj", "model.layers.20.mlp.experts.25.up_proj", "model.layers.20.mlp.experts.26.up_proj", "model.layers.20.mlp.experts.27.up_proj", "model.layers.20.mlp.experts.28.up_proj", "model.layers.20.mlp.experts.29.up_proj", "model.layers.20.mlp.experts.30.up_proj", "model.layers.20.mlp.experts.31.up_proj", "model.layers.20.mlp.experts.32.up_proj", "model.layers.20.mlp.experts.33.up_proj", "model.layers.20.mlp.experts.34.up_proj", "model.layers.20.mlp.experts.35.up_proj", "model.layers.20.mlp.experts.36.up_proj", "model.layers.20.mlp.experts.37.up_proj", "model.layers.20.mlp.experts.38.up_proj", "model.layers.20.mlp.experts.39.up_proj", "model.layers.20.mlp.experts.40.up_proj", "model.layers.20.mlp.experts.41.up_proj", "model.layers.20.mlp.experts.42.up_proj", "model.layers.20.mlp.experts.43.up_proj", "model.layers.20.mlp.experts.44.up_proj", "model.layers.20.mlp.experts.45.up_proj", "model.layers.20.mlp.experts.46.up_proj", "model.layers.20.mlp.experts.47.up_proj", "model.layers.20.mlp.experts.48.up_proj", "model.layers.20.mlp.experts.49.up_proj", "model.layers.20.mlp.experts.50.up_proj", "model.layers.20.mlp.experts.51.up_proj", "model.layers.20.mlp.experts.52.up_proj", "model.layers.20.mlp.experts.53.up_proj", "model.layers.20.mlp.experts.54.up_proj", "model.layers.20.mlp.experts.55.up_proj", "model.layers.20.mlp.experts.56.up_proj", "model.layers.20.mlp.experts.57.up_proj", "model.layers.20.mlp.experts.58.up_proj", "model.layers.20.mlp.experts.59.up_proj", "model.layers.20.mlp.experts.60.up_proj", "model.layers.20.mlp.experts.61.up_proj", "model.layers.20.mlp.experts.62.up_proj", "model.layers.20.mlp.experts.63.up_proj", "model.layers.20.mlp.experts.64.up_proj", "model.layers.20.mlp.experts.65.up_proj", "model.layers.20.mlp.experts.66.up_proj", "model.layers.20.mlp.experts.67.up_proj", "model.layers.20.mlp.experts.68.up_proj", "model.layers.20.mlp.experts.69.up_proj", "model.layers.20.mlp.experts.70.up_proj", "model.layers.20.mlp.experts.71.up_proj", "model.layers.20.mlp.experts.72.up_proj", "model.layers.20.mlp.experts.73.up_proj", "model.layers.20.mlp.experts.74.up_proj", "model.layers.20.mlp.experts.75.up_proj", "model.layers.20.mlp.experts.76.up_proj", "model.layers.20.mlp.experts.77.up_proj", "model.layers.20.mlp.experts.78.up_proj", "model.layers.20.mlp.experts.79.up_proj", "model.layers.20.mlp.experts.80.up_proj", "model.layers.20.mlp.experts.81.up_proj", "model.layers.20.mlp.experts.82.up_proj", "model.layers.20.mlp.experts.83.up_proj", "model.layers.20.mlp.experts.84.up_proj", "model.layers.20.mlp.experts.85.up_proj", "model.layers.20.mlp.experts.86.up_proj", "model.layers.20.mlp.experts.87.up_proj", "model.layers.20.mlp.experts.88.up_proj", "model.layers.20.mlp.experts.89.up_proj", "model.layers.20.mlp.experts.90.up_proj", "model.layers.20.mlp.experts.91.up_proj", "model.layers.20.mlp.experts.92.up_proj", "model.layers.20.mlp.experts.93.up_proj", "model.layers.20.mlp.experts.94.up_proj", "model.layers.20.mlp.experts.95.up_proj", "model.layers.20.mlp.experts.96.up_proj", "model.layers.20.mlp.experts.97.up_proj", "model.layers.20.mlp.experts.98.up_proj", "model.layers.20.mlp.experts.99.up_proj", "model.layers.20.mlp.experts.100.up_proj", "model.layers.20.mlp.experts.101.up_proj", "model.layers.20.mlp.experts.102.up_proj", "model.layers.20.mlp.experts.103.up_proj", "model.layers.20.mlp.experts.104.up_proj", "model.layers.20.mlp.experts.105.up_proj", "model.layers.20.mlp.experts.106.up_proj", "model.layers.20.mlp.experts.107.up_proj", "model.layers.20.mlp.experts.108.up_proj", "model.layers.20.mlp.experts.109.up_proj", "model.layers.20.mlp.experts.110.up_proj", "model.layers.20.mlp.experts.111.up_proj", "model.layers.20.mlp.experts.112.up_proj", "model.layers.20.mlp.experts.113.up_proj", "model.layers.20.mlp.experts.114.up_proj", "model.layers.20.mlp.experts.115.up_proj", "model.layers.20.mlp.experts.116.up_proj", "model.layers.20.mlp.experts.117.up_proj", "model.layers.20.mlp.experts.118.up_proj", "model.layers.20.mlp.experts.119.up_proj", "model.layers.20.mlp.experts.120.up_proj", "model.layers.20.mlp.experts.121.up_proj", "model.layers.20.mlp.experts.122.up_proj", "model.layers.20.mlp.experts.123.up_proj", "model.layers.20.mlp.experts.124.up_proj", "model.layers.20.mlp.experts.125.up_proj", "model.layers.20.mlp.experts.126.up_proj", "model.layers.20.mlp.experts.127.up_proj", "model.layers.20.mlp.experts.128.up_proj", "model.layers.20.mlp.experts.129.up_proj", "model.layers.20.mlp.experts.130.up_proj", "model.layers.20.mlp.experts.131.up_proj", "model.layers.20.mlp.experts.132.up_proj", "model.layers.20.mlp.experts.133.up_proj", "model.layers.20.mlp.experts.134.up_proj", "model.layers.20.mlp.experts.135.up_proj", "model.layers.20.mlp.experts.136.up_proj", "model.layers.20.mlp.experts.137.up_proj", "model.layers.20.mlp.experts.138.up_proj", "model.layers.20.mlp.experts.139.up_proj", "model.layers.20.mlp.experts.140.up_proj", "model.layers.20.mlp.experts.141.up_proj", "model.layers.20.mlp.experts.142.up_proj", "model.layers.20.mlp.experts.143.up_proj", "model.layers.20.mlp.experts.144.up_proj", "model.layers.20.mlp.experts.145.up_proj", "model.layers.20.mlp.experts.146.up_proj", "model.layers.20.mlp.experts.147.up_proj", "model.layers.20.mlp.experts.148.up_proj", "model.layers.20.mlp.experts.149.up_proj", "model.layers.20.mlp.experts.150.up_proj", "model.layers.20.mlp.experts.151.up_proj", "model.layers.20.mlp.experts.152.up_proj", "model.layers.20.mlp.experts.153.up_proj", "model.layers.20.mlp.experts.154.up_proj", "model.layers.20.mlp.experts.155.up_proj", "model.layers.20.mlp.experts.156.up_proj", "model.layers.20.mlp.experts.157.up_proj", "model.layers.20.mlp.experts.158.up_proj", "model.layers.20.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.0443007340654297e-06, "dbits": 5033164800 } ] }, { "idx": 122, "layers": [ "model.layers.20.mlp.experts.0.down_proj", "model.layers.20.mlp.experts.1.down_proj", "model.layers.20.mlp.experts.2.down_proj", "model.layers.20.mlp.experts.3.down_proj", "model.layers.20.mlp.experts.4.down_proj", "model.layers.20.mlp.experts.5.down_proj", "model.layers.20.mlp.experts.6.down_proj", "model.layers.20.mlp.experts.7.down_proj", "model.layers.20.mlp.experts.8.down_proj", "model.layers.20.mlp.experts.9.down_proj", "model.layers.20.mlp.experts.10.down_proj", "model.layers.20.mlp.experts.11.down_proj", "model.layers.20.mlp.experts.12.down_proj", "model.layers.20.mlp.experts.13.down_proj", "model.layers.20.mlp.experts.14.down_proj", "model.layers.20.mlp.experts.15.down_proj", "model.layers.20.mlp.experts.16.down_proj", "model.layers.20.mlp.experts.17.down_proj", "model.layers.20.mlp.experts.18.down_proj", "model.layers.20.mlp.experts.19.down_proj", "model.layers.20.mlp.experts.20.down_proj", "model.layers.20.mlp.experts.21.down_proj", "model.layers.20.mlp.experts.22.down_proj", "model.layers.20.mlp.experts.23.down_proj", "model.layers.20.mlp.experts.24.down_proj", "model.layers.20.mlp.experts.25.down_proj", "model.layers.20.mlp.experts.26.down_proj", "model.layers.20.mlp.experts.27.down_proj", "model.layers.20.mlp.experts.28.down_proj", "model.layers.20.mlp.experts.29.down_proj", "model.layers.20.mlp.experts.30.down_proj", "model.layers.20.mlp.experts.31.down_proj", "model.layers.20.mlp.experts.32.down_proj", "model.layers.20.mlp.experts.33.down_proj", "model.layers.20.mlp.experts.34.down_proj", "model.layers.20.mlp.experts.35.down_proj", "model.layers.20.mlp.experts.36.down_proj", "model.layers.20.mlp.experts.37.down_proj", "model.layers.20.mlp.experts.38.down_proj", "model.layers.20.mlp.experts.39.down_proj", "model.layers.20.mlp.experts.40.down_proj", "model.layers.20.mlp.experts.41.down_proj", "model.layers.20.mlp.experts.42.down_proj", "model.layers.20.mlp.experts.43.down_proj", "model.layers.20.mlp.experts.44.down_proj", "model.layers.20.mlp.experts.45.down_proj", "model.layers.20.mlp.experts.46.down_proj", "model.layers.20.mlp.experts.47.down_proj", "model.layers.20.mlp.experts.48.down_proj", "model.layers.20.mlp.experts.49.down_proj", "model.layers.20.mlp.experts.50.down_proj", "model.layers.20.mlp.experts.51.down_proj", "model.layers.20.mlp.experts.52.down_proj", "model.layers.20.mlp.experts.53.down_proj", "model.layers.20.mlp.experts.54.down_proj", "model.layers.20.mlp.experts.55.down_proj", "model.layers.20.mlp.experts.56.down_proj", "model.layers.20.mlp.experts.57.down_proj", "model.layers.20.mlp.experts.58.down_proj", "model.layers.20.mlp.experts.59.down_proj", "model.layers.20.mlp.experts.60.down_proj", "model.layers.20.mlp.experts.61.down_proj", "model.layers.20.mlp.experts.62.down_proj", "model.layers.20.mlp.experts.63.down_proj", "model.layers.20.mlp.experts.64.down_proj", "model.layers.20.mlp.experts.65.down_proj", "model.layers.20.mlp.experts.66.down_proj", "model.layers.20.mlp.experts.67.down_proj", "model.layers.20.mlp.experts.68.down_proj", "model.layers.20.mlp.experts.69.down_proj", "model.layers.20.mlp.experts.70.down_proj", "model.layers.20.mlp.experts.71.down_proj", "model.layers.20.mlp.experts.72.down_proj", "model.layers.20.mlp.experts.73.down_proj", "model.layers.20.mlp.experts.74.down_proj", "model.layers.20.mlp.experts.75.down_proj", "model.layers.20.mlp.experts.76.down_proj", "model.layers.20.mlp.experts.77.down_proj", "model.layers.20.mlp.experts.78.down_proj", "model.layers.20.mlp.experts.79.down_proj", "model.layers.20.mlp.experts.80.down_proj", "model.layers.20.mlp.experts.81.down_proj", "model.layers.20.mlp.experts.82.down_proj", "model.layers.20.mlp.experts.83.down_proj", "model.layers.20.mlp.experts.84.down_proj", "model.layers.20.mlp.experts.85.down_proj", "model.layers.20.mlp.experts.86.down_proj", "model.layers.20.mlp.experts.87.down_proj", "model.layers.20.mlp.experts.88.down_proj", "model.layers.20.mlp.experts.89.down_proj", "model.layers.20.mlp.experts.90.down_proj", "model.layers.20.mlp.experts.91.down_proj", "model.layers.20.mlp.experts.92.down_proj", "model.layers.20.mlp.experts.93.down_proj", "model.layers.20.mlp.experts.94.down_proj", "model.layers.20.mlp.experts.95.down_proj", "model.layers.20.mlp.experts.96.down_proj", "model.layers.20.mlp.experts.97.down_proj", "model.layers.20.mlp.experts.98.down_proj", "model.layers.20.mlp.experts.99.down_proj", "model.layers.20.mlp.experts.100.down_proj", "model.layers.20.mlp.experts.101.down_proj", "model.layers.20.mlp.experts.102.down_proj", "model.layers.20.mlp.experts.103.down_proj", "model.layers.20.mlp.experts.104.down_proj", "model.layers.20.mlp.experts.105.down_proj", "model.layers.20.mlp.experts.106.down_proj", "model.layers.20.mlp.experts.107.down_proj", "model.layers.20.mlp.experts.108.down_proj", "model.layers.20.mlp.experts.109.down_proj", "model.layers.20.mlp.experts.110.down_proj", "model.layers.20.mlp.experts.111.down_proj", "model.layers.20.mlp.experts.112.down_proj", "model.layers.20.mlp.experts.113.down_proj", "model.layers.20.mlp.experts.114.down_proj", "model.layers.20.mlp.experts.115.down_proj", "model.layers.20.mlp.experts.116.down_proj", "model.layers.20.mlp.experts.117.down_proj", "model.layers.20.mlp.experts.118.down_proj", "model.layers.20.mlp.experts.119.down_proj", "model.layers.20.mlp.experts.120.down_proj", "model.layers.20.mlp.experts.121.down_proj", "model.layers.20.mlp.experts.122.down_proj", "model.layers.20.mlp.experts.123.down_proj", "model.layers.20.mlp.experts.124.down_proj", "model.layers.20.mlp.experts.125.down_proj", "model.layers.20.mlp.experts.126.down_proj", "model.layers.20.mlp.experts.127.down_proj", "model.layers.20.mlp.experts.128.down_proj", "model.layers.20.mlp.experts.129.down_proj", "model.layers.20.mlp.experts.130.down_proj", "model.layers.20.mlp.experts.131.down_proj", "model.layers.20.mlp.experts.132.down_proj", "model.layers.20.mlp.experts.133.down_proj", "model.layers.20.mlp.experts.134.down_proj", "model.layers.20.mlp.experts.135.down_proj", "model.layers.20.mlp.experts.136.down_proj", "model.layers.20.mlp.experts.137.down_proj", "model.layers.20.mlp.experts.138.down_proj", "model.layers.20.mlp.experts.139.down_proj", "model.layers.20.mlp.experts.140.down_proj", "model.layers.20.mlp.experts.141.down_proj", "model.layers.20.mlp.experts.142.down_proj", "model.layers.20.mlp.experts.143.down_proj", "model.layers.20.mlp.experts.144.down_proj", "model.layers.20.mlp.experts.145.down_proj", "model.layers.20.mlp.experts.146.down_proj", "model.layers.20.mlp.experts.147.down_proj", "model.layers.20.mlp.experts.148.down_proj", "model.layers.20.mlp.experts.149.down_proj", "model.layers.20.mlp.experts.150.down_proj", "model.layers.20.mlp.experts.151.down_proj", "model.layers.20.mlp.experts.152.down_proj", "model.layers.20.mlp.experts.153.down_proj", "model.layers.20.mlp.experts.154.down_proj", "model.layers.20.mlp.experts.155.down_proj", "model.layers.20.mlp.experts.156.down_proj", "model.layers.20.mlp.experts.157.down_proj", "model.layers.20.mlp.experts.158.down_proj", "model.layers.20.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.0336429113521955e-06, "dbits": 2516582400 } ] }, { "idx": 123, "layers": [ "model.layers.21.self_attn.q_proj" ], "candidates": [ { "dkld": -2.318053448107098e-05, "dbits": 125829120 } ] }, { "idx": 124, "layers": [ "model.layers.21.self_attn.k_proj", "model.layers.21.self_attn.v_proj" ], "candidates": [ { "dkld": 3.0735123436897906e-05, "dbits": 20971520 } ] }, { "idx": 125, "layers": [ "model.layers.21.self_attn.o_proj" ], "candidates": [ { "dkld": 5.945190787315455e-06, "dbits": 125829120 } ] }, { "idx": 126, "layers": [ "model.layers.21.mlp.shared_experts.gate_proj", "model.layers.21.mlp.shared_experts.up_proj", "model.layers.21.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.683201116975397e-05, "dbits": 47185920 } ] }, { "idx": 127, "layers": [ "model.layers.21.mlp.experts.0.gate_proj", "model.layers.21.mlp.experts.1.gate_proj", "model.layers.21.mlp.experts.2.gate_proj", "model.layers.21.mlp.experts.3.gate_proj", "model.layers.21.mlp.experts.4.gate_proj", "model.layers.21.mlp.experts.5.gate_proj", "model.layers.21.mlp.experts.6.gate_proj", "model.layers.21.mlp.experts.7.gate_proj", "model.layers.21.mlp.experts.8.gate_proj", "model.layers.21.mlp.experts.9.gate_proj", "model.layers.21.mlp.experts.10.gate_proj", "model.layers.21.mlp.experts.11.gate_proj", "model.layers.21.mlp.experts.12.gate_proj", "model.layers.21.mlp.experts.13.gate_proj", "model.layers.21.mlp.experts.14.gate_proj", "model.layers.21.mlp.experts.15.gate_proj", "model.layers.21.mlp.experts.16.gate_proj", "model.layers.21.mlp.experts.17.gate_proj", "model.layers.21.mlp.experts.18.gate_proj", "model.layers.21.mlp.experts.19.gate_proj", "model.layers.21.mlp.experts.20.gate_proj", "model.layers.21.mlp.experts.21.gate_proj", "model.layers.21.mlp.experts.22.gate_proj", "model.layers.21.mlp.experts.23.gate_proj", "model.layers.21.mlp.experts.24.gate_proj", "model.layers.21.mlp.experts.25.gate_proj", "model.layers.21.mlp.experts.26.gate_proj", "model.layers.21.mlp.experts.27.gate_proj", "model.layers.21.mlp.experts.28.gate_proj", "model.layers.21.mlp.experts.29.gate_proj", "model.layers.21.mlp.experts.30.gate_proj", "model.layers.21.mlp.experts.31.gate_proj", "model.layers.21.mlp.experts.32.gate_proj", "model.layers.21.mlp.experts.33.gate_proj", "model.layers.21.mlp.experts.34.gate_proj", "model.layers.21.mlp.experts.35.gate_proj", "model.layers.21.mlp.experts.36.gate_proj", "model.layers.21.mlp.experts.37.gate_proj", "model.layers.21.mlp.experts.38.gate_proj", "model.layers.21.mlp.experts.39.gate_proj", "model.layers.21.mlp.experts.40.gate_proj", "model.layers.21.mlp.experts.41.gate_proj", "model.layers.21.mlp.experts.42.gate_proj", "model.layers.21.mlp.experts.43.gate_proj", "model.layers.21.mlp.experts.44.gate_proj", "model.layers.21.mlp.experts.45.gate_proj", "model.layers.21.mlp.experts.46.gate_proj", "model.layers.21.mlp.experts.47.gate_proj", "model.layers.21.mlp.experts.48.gate_proj", "model.layers.21.mlp.experts.49.gate_proj", "model.layers.21.mlp.experts.50.gate_proj", "model.layers.21.mlp.experts.51.gate_proj", "model.layers.21.mlp.experts.52.gate_proj", "model.layers.21.mlp.experts.53.gate_proj", "model.layers.21.mlp.experts.54.gate_proj", "model.layers.21.mlp.experts.55.gate_proj", "model.layers.21.mlp.experts.56.gate_proj", "model.layers.21.mlp.experts.57.gate_proj", "model.layers.21.mlp.experts.58.gate_proj", "model.layers.21.mlp.experts.59.gate_proj", "model.layers.21.mlp.experts.60.gate_proj", "model.layers.21.mlp.experts.61.gate_proj", "model.layers.21.mlp.experts.62.gate_proj", "model.layers.21.mlp.experts.63.gate_proj", "model.layers.21.mlp.experts.64.gate_proj", "model.layers.21.mlp.experts.65.gate_proj", "model.layers.21.mlp.experts.66.gate_proj", "model.layers.21.mlp.experts.67.gate_proj", "model.layers.21.mlp.experts.68.gate_proj", "model.layers.21.mlp.experts.69.gate_proj", "model.layers.21.mlp.experts.70.gate_proj", "model.layers.21.mlp.experts.71.gate_proj", "model.layers.21.mlp.experts.72.gate_proj", "model.layers.21.mlp.experts.73.gate_proj", "model.layers.21.mlp.experts.74.gate_proj", "model.layers.21.mlp.experts.75.gate_proj", "model.layers.21.mlp.experts.76.gate_proj", "model.layers.21.mlp.experts.77.gate_proj", "model.layers.21.mlp.experts.78.gate_proj", "model.layers.21.mlp.experts.79.gate_proj", "model.layers.21.mlp.experts.80.gate_proj", "model.layers.21.mlp.experts.81.gate_proj", "model.layers.21.mlp.experts.82.gate_proj", "model.layers.21.mlp.experts.83.gate_proj", "model.layers.21.mlp.experts.84.gate_proj", "model.layers.21.mlp.experts.85.gate_proj", "model.layers.21.mlp.experts.86.gate_proj", "model.layers.21.mlp.experts.87.gate_proj", "model.layers.21.mlp.experts.88.gate_proj", "model.layers.21.mlp.experts.89.gate_proj", "model.layers.21.mlp.experts.90.gate_proj", "model.layers.21.mlp.experts.91.gate_proj", "model.layers.21.mlp.experts.92.gate_proj", "model.layers.21.mlp.experts.93.gate_proj", "model.layers.21.mlp.experts.94.gate_proj", "model.layers.21.mlp.experts.95.gate_proj", "model.layers.21.mlp.experts.96.gate_proj", "model.layers.21.mlp.experts.97.gate_proj", "model.layers.21.mlp.experts.98.gate_proj", "model.layers.21.mlp.experts.99.gate_proj", "model.layers.21.mlp.experts.100.gate_proj", "model.layers.21.mlp.experts.101.gate_proj", "model.layers.21.mlp.experts.102.gate_proj", "model.layers.21.mlp.experts.103.gate_proj", "model.layers.21.mlp.experts.104.gate_proj", "model.layers.21.mlp.experts.105.gate_proj", "model.layers.21.mlp.experts.106.gate_proj", "model.layers.21.mlp.experts.107.gate_proj", "model.layers.21.mlp.experts.108.gate_proj", "model.layers.21.mlp.experts.109.gate_proj", "model.layers.21.mlp.experts.110.gate_proj", "model.layers.21.mlp.experts.111.gate_proj", "model.layers.21.mlp.experts.112.gate_proj", "model.layers.21.mlp.experts.113.gate_proj", "model.layers.21.mlp.experts.114.gate_proj", "model.layers.21.mlp.experts.115.gate_proj", "model.layers.21.mlp.experts.116.gate_proj", "model.layers.21.mlp.experts.117.gate_proj", "model.layers.21.mlp.experts.118.gate_proj", "model.layers.21.mlp.experts.119.gate_proj", "model.layers.21.mlp.experts.120.gate_proj", "model.layers.21.mlp.experts.121.gate_proj", "model.layers.21.mlp.experts.122.gate_proj", "model.layers.21.mlp.experts.123.gate_proj", "model.layers.21.mlp.experts.124.gate_proj", "model.layers.21.mlp.experts.125.gate_proj", "model.layers.21.mlp.experts.126.gate_proj", "model.layers.21.mlp.experts.127.gate_proj", "model.layers.21.mlp.experts.128.gate_proj", "model.layers.21.mlp.experts.129.gate_proj", "model.layers.21.mlp.experts.130.gate_proj", "model.layers.21.mlp.experts.131.gate_proj", "model.layers.21.mlp.experts.132.gate_proj", "model.layers.21.mlp.experts.133.gate_proj", "model.layers.21.mlp.experts.134.gate_proj", "model.layers.21.mlp.experts.135.gate_proj", "model.layers.21.mlp.experts.136.gate_proj", "model.layers.21.mlp.experts.137.gate_proj", "model.layers.21.mlp.experts.138.gate_proj", "model.layers.21.mlp.experts.139.gate_proj", "model.layers.21.mlp.experts.140.gate_proj", "model.layers.21.mlp.experts.141.gate_proj", "model.layers.21.mlp.experts.142.gate_proj", "model.layers.21.mlp.experts.143.gate_proj", "model.layers.21.mlp.experts.144.gate_proj", "model.layers.21.mlp.experts.145.gate_proj", "model.layers.21.mlp.experts.146.gate_proj", "model.layers.21.mlp.experts.147.gate_proj", "model.layers.21.mlp.experts.148.gate_proj", "model.layers.21.mlp.experts.149.gate_proj", "model.layers.21.mlp.experts.150.gate_proj", "model.layers.21.mlp.experts.151.gate_proj", "model.layers.21.mlp.experts.152.gate_proj", "model.layers.21.mlp.experts.153.gate_proj", "model.layers.21.mlp.experts.154.gate_proj", "model.layers.21.mlp.experts.155.gate_proj", "model.layers.21.mlp.experts.156.gate_proj", "model.layers.21.mlp.experts.157.gate_proj", "model.layers.21.mlp.experts.158.gate_proj", "model.layers.21.mlp.experts.159.gate_proj", "model.layers.21.mlp.experts.0.up_proj", "model.layers.21.mlp.experts.1.up_proj", "model.layers.21.mlp.experts.2.up_proj", "model.layers.21.mlp.experts.3.up_proj", "model.layers.21.mlp.experts.4.up_proj", "model.layers.21.mlp.experts.5.up_proj", "model.layers.21.mlp.experts.6.up_proj", "model.layers.21.mlp.experts.7.up_proj", "model.layers.21.mlp.experts.8.up_proj", "model.layers.21.mlp.experts.9.up_proj", "model.layers.21.mlp.experts.10.up_proj", "model.layers.21.mlp.experts.11.up_proj", "model.layers.21.mlp.experts.12.up_proj", "model.layers.21.mlp.experts.13.up_proj", "model.layers.21.mlp.experts.14.up_proj", "model.layers.21.mlp.experts.15.up_proj", "model.layers.21.mlp.experts.16.up_proj", "model.layers.21.mlp.experts.17.up_proj", "model.layers.21.mlp.experts.18.up_proj", "model.layers.21.mlp.experts.19.up_proj", "model.layers.21.mlp.experts.20.up_proj", "model.layers.21.mlp.experts.21.up_proj", "model.layers.21.mlp.experts.22.up_proj", "model.layers.21.mlp.experts.23.up_proj", "model.layers.21.mlp.experts.24.up_proj", "model.layers.21.mlp.experts.25.up_proj", "model.layers.21.mlp.experts.26.up_proj", "model.layers.21.mlp.experts.27.up_proj", "model.layers.21.mlp.experts.28.up_proj", "model.layers.21.mlp.experts.29.up_proj", "model.layers.21.mlp.experts.30.up_proj", "model.layers.21.mlp.experts.31.up_proj", "model.layers.21.mlp.experts.32.up_proj", "model.layers.21.mlp.experts.33.up_proj", "model.layers.21.mlp.experts.34.up_proj", "model.layers.21.mlp.experts.35.up_proj", "model.layers.21.mlp.experts.36.up_proj", "model.layers.21.mlp.experts.37.up_proj", "model.layers.21.mlp.experts.38.up_proj", "model.layers.21.mlp.experts.39.up_proj", "model.layers.21.mlp.experts.40.up_proj", "model.layers.21.mlp.experts.41.up_proj", "model.layers.21.mlp.experts.42.up_proj", "model.layers.21.mlp.experts.43.up_proj", "model.layers.21.mlp.experts.44.up_proj", "model.layers.21.mlp.experts.45.up_proj", "model.layers.21.mlp.experts.46.up_proj", "model.layers.21.mlp.experts.47.up_proj", "model.layers.21.mlp.experts.48.up_proj", "model.layers.21.mlp.experts.49.up_proj", "model.layers.21.mlp.experts.50.up_proj", "model.layers.21.mlp.experts.51.up_proj", "model.layers.21.mlp.experts.52.up_proj", "model.layers.21.mlp.experts.53.up_proj", "model.layers.21.mlp.experts.54.up_proj", "model.layers.21.mlp.experts.55.up_proj", "model.layers.21.mlp.experts.56.up_proj", "model.layers.21.mlp.experts.57.up_proj", "model.layers.21.mlp.experts.58.up_proj", "model.layers.21.mlp.experts.59.up_proj", "model.layers.21.mlp.experts.60.up_proj", "model.layers.21.mlp.experts.61.up_proj", "model.layers.21.mlp.experts.62.up_proj", "model.layers.21.mlp.experts.63.up_proj", "model.layers.21.mlp.experts.64.up_proj", "model.layers.21.mlp.experts.65.up_proj", "model.layers.21.mlp.experts.66.up_proj", "model.layers.21.mlp.experts.67.up_proj", "model.layers.21.mlp.experts.68.up_proj", "model.layers.21.mlp.experts.69.up_proj", "model.layers.21.mlp.experts.70.up_proj", "model.layers.21.mlp.experts.71.up_proj", "model.layers.21.mlp.experts.72.up_proj", "model.layers.21.mlp.experts.73.up_proj", "model.layers.21.mlp.experts.74.up_proj", "model.layers.21.mlp.experts.75.up_proj", "model.layers.21.mlp.experts.76.up_proj", "model.layers.21.mlp.experts.77.up_proj", "model.layers.21.mlp.experts.78.up_proj", "model.layers.21.mlp.experts.79.up_proj", "model.layers.21.mlp.experts.80.up_proj", "model.layers.21.mlp.experts.81.up_proj", "model.layers.21.mlp.experts.82.up_proj", "model.layers.21.mlp.experts.83.up_proj", "model.layers.21.mlp.experts.84.up_proj", "model.layers.21.mlp.experts.85.up_proj", "model.layers.21.mlp.experts.86.up_proj", "model.layers.21.mlp.experts.87.up_proj", "model.layers.21.mlp.experts.88.up_proj", "model.layers.21.mlp.experts.89.up_proj", "model.layers.21.mlp.experts.90.up_proj", "model.layers.21.mlp.experts.91.up_proj", "model.layers.21.mlp.experts.92.up_proj", "model.layers.21.mlp.experts.93.up_proj", "model.layers.21.mlp.experts.94.up_proj", "model.layers.21.mlp.experts.95.up_proj", "model.layers.21.mlp.experts.96.up_proj", "model.layers.21.mlp.experts.97.up_proj", "model.layers.21.mlp.experts.98.up_proj", "model.layers.21.mlp.experts.99.up_proj", "model.layers.21.mlp.experts.100.up_proj", "model.layers.21.mlp.experts.101.up_proj", "model.layers.21.mlp.experts.102.up_proj", "model.layers.21.mlp.experts.103.up_proj", "model.layers.21.mlp.experts.104.up_proj", "model.layers.21.mlp.experts.105.up_proj", "model.layers.21.mlp.experts.106.up_proj", "model.layers.21.mlp.experts.107.up_proj", "model.layers.21.mlp.experts.108.up_proj", "model.layers.21.mlp.experts.109.up_proj", "model.layers.21.mlp.experts.110.up_proj", "model.layers.21.mlp.experts.111.up_proj", "model.layers.21.mlp.experts.112.up_proj", "model.layers.21.mlp.experts.113.up_proj", "model.layers.21.mlp.experts.114.up_proj", "model.layers.21.mlp.experts.115.up_proj", "model.layers.21.mlp.experts.116.up_proj", "model.layers.21.mlp.experts.117.up_proj", "model.layers.21.mlp.experts.118.up_proj", "model.layers.21.mlp.experts.119.up_proj", "model.layers.21.mlp.experts.120.up_proj", "model.layers.21.mlp.experts.121.up_proj", "model.layers.21.mlp.experts.122.up_proj", "model.layers.21.mlp.experts.123.up_proj", "model.layers.21.mlp.experts.124.up_proj", "model.layers.21.mlp.experts.125.up_proj", "model.layers.21.mlp.experts.126.up_proj", "model.layers.21.mlp.experts.127.up_proj", "model.layers.21.mlp.experts.128.up_proj", "model.layers.21.mlp.experts.129.up_proj", "model.layers.21.mlp.experts.130.up_proj", "model.layers.21.mlp.experts.131.up_proj", "model.layers.21.mlp.experts.132.up_proj", "model.layers.21.mlp.experts.133.up_proj", "model.layers.21.mlp.experts.134.up_proj", "model.layers.21.mlp.experts.135.up_proj", "model.layers.21.mlp.experts.136.up_proj", "model.layers.21.mlp.experts.137.up_proj", "model.layers.21.mlp.experts.138.up_proj", "model.layers.21.mlp.experts.139.up_proj", "model.layers.21.mlp.experts.140.up_proj", "model.layers.21.mlp.experts.141.up_proj", "model.layers.21.mlp.experts.142.up_proj", "model.layers.21.mlp.experts.143.up_proj", "model.layers.21.mlp.experts.144.up_proj", "model.layers.21.mlp.experts.145.up_proj", "model.layers.21.mlp.experts.146.up_proj", "model.layers.21.mlp.experts.147.up_proj", "model.layers.21.mlp.experts.148.up_proj", "model.layers.21.mlp.experts.149.up_proj", "model.layers.21.mlp.experts.150.up_proj", "model.layers.21.mlp.experts.151.up_proj", "model.layers.21.mlp.experts.152.up_proj", "model.layers.21.mlp.experts.153.up_proj", "model.layers.21.mlp.experts.154.up_proj", "model.layers.21.mlp.experts.155.up_proj", "model.layers.21.mlp.experts.156.up_proj", "model.layers.21.mlp.experts.157.up_proj", "model.layers.21.mlp.experts.158.up_proj", "model.layers.21.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 9.341814438812348e-06, "dbits": 5033164800 } ] }, { "idx": 128, "layers": [ "model.layers.21.mlp.experts.0.down_proj", "model.layers.21.mlp.experts.1.down_proj", "model.layers.21.mlp.experts.2.down_proj", "model.layers.21.mlp.experts.3.down_proj", "model.layers.21.mlp.experts.4.down_proj", "model.layers.21.mlp.experts.5.down_proj", "model.layers.21.mlp.experts.6.down_proj", "model.layers.21.mlp.experts.7.down_proj", "model.layers.21.mlp.experts.8.down_proj", "model.layers.21.mlp.experts.9.down_proj", "model.layers.21.mlp.experts.10.down_proj", "model.layers.21.mlp.experts.11.down_proj", "model.layers.21.mlp.experts.12.down_proj", "model.layers.21.mlp.experts.13.down_proj", "model.layers.21.mlp.experts.14.down_proj", "model.layers.21.mlp.experts.15.down_proj", "model.layers.21.mlp.experts.16.down_proj", "model.layers.21.mlp.experts.17.down_proj", "model.layers.21.mlp.experts.18.down_proj", "model.layers.21.mlp.experts.19.down_proj", "model.layers.21.mlp.experts.20.down_proj", "model.layers.21.mlp.experts.21.down_proj", "model.layers.21.mlp.experts.22.down_proj", "model.layers.21.mlp.experts.23.down_proj", "model.layers.21.mlp.experts.24.down_proj", "model.layers.21.mlp.experts.25.down_proj", "model.layers.21.mlp.experts.26.down_proj", "model.layers.21.mlp.experts.27.down_proj", "model.layers.21.mlp.experts.28.down_proj", "model.layers.21.mlp.experts.29.down_proj", "model.layers.21.mlp.experts.30.down_proj", "model.layers.21.mlp.experts.31.down_proj", "model.layers.21.mlp.experts.32.down_proj", "model.layers.21.mlp.experts.33.down_proj", "model.layers.21.mlp.experts.34.down_proj", "model.layers.21.mlp.experts.35.down_proj", "model.layers.21.mlp.experts.36.down_proj", "model.layers.21.mlp.experts.37.down_proj", "model.layers.21.mlp.experts.38.down_proj", "model.layers.21.mlp.experts.39.down_proj", "model.layers.21.mlp.experts.40.down_proj", "model.layers.21.mlp.experts.41.down_proj", "model.layers.21.mlp.experts.42.down_proj", "model.layers.21.mlp.experts.43.down_proj", "model.layers.21.mlp.experts.44.down_proj", "model.layers.21.mlp.experts.45.down_proj", "model.layers.21.mlp.experts.46.down_proj", "model.layers.21.mlp.experts.47.down_proj", "model.layers.21.mlp.experts.48.down_proj", "model.layers.21.mlp.experts.49.down_proj", "model.layers.21.mlp.experts.50.down_proj", "model.layers.21.mlp.experts.51.down_proj", "model.layers.21.mlp.experts.52.down_proj", "model.layers.21.mlp.experts.53.down_proj", "model.layers.21.mlp.experts.54.down_proj", "model.layers.21.mlp.experts.55.down_proj", "model.layers.21.mlp.experts.56.down_proj", "model.layers.21.mlp.experts.57.down_proj", "model.layers.21.mlp.experts.58.down_proj", "model.layers.21.mlp.experts.59.down_proj", "model.layers.21.mlp.experts.60.down_proj", "model.layers.21.mlp.experts.61.down_proj", "model.layers.21.mlp.experts.62.down_proj", "model.layers.21.mlp.experts.63.down_proj", "model.layers.21.mlp.experts.64.down_proj", "model.layers.21.mlp.experts.65.down_proj", "model.layers.21.mlp.experts.66.down_proj", "model.layers.21.mlp.experts.67.down_proj", "model.layers.21.mlp.experts.68.down_proj", "model.layers.21.mlp.experts.69.down_proj", "model.layers.21.mlp.experts.70.down_proj", "model.layers.21.mlp.experts.71.down_proj", "model.layers.21.mlp.experts.72.down_proj", "model.layers.21.mlp.experts.73.down_proj", "model.layers.21.mlp.experts.74.down_proj", "model.layers.21.mlp.experts.75.down_proj", "model.layers.21.mlp.experts.76.down_proj", "model.layers.21.mlp.experts.77.down_proj", "model.layers.21.mlp.experts.78.down_proj", "model.layers.21.mlp.experts.79.down_proj", "model.layers.21.mlp.experts.80.down_proj", "model.layers.21.mlp.experts.81.down_proj", "model.layers.21.mlp.experts.82.down_proj", "model.layers.21.mlp.experts.83.down_proj", "model.layers.21.mlp.experts.84.down_proj", "model.layers.21.mlp.experts.85.down_proj", "model.layers.21.mlp.experts.86.down_proj", "model.layers.21.mlp.experts.87.down_proj", "model.layers.21.mlp.experts.88.down_proj", "model.layers.21.mlp.experts.89.down_proj", "model.layers.21.mlp.experts.90.down_proj", "model.layers.21.mlp.experts.91.down_proj", "model.layers.21.mlp.experts.92.down_proj", "model.layers.21.mlp.experts.93.down_proj", "model.layers.21.mlp.experts.94.down_proj", "model.layers.21.mlp.experts.95.down_proj", "model.layers.21.mlp.experts.96.down_proj", "model.layers.21.mlp.experts.97.down_proj", "model.layers.21.mlp.experts.98.down_proj", "model.layers.21.mlp.experts.99.down_proj", "model.layers.21.mlp.experts.100.down_proj", "model.layers.21.mlp.experts.101.down_proj", "model.layers.21.mlp.experts.102.down_proj", "model.layers.21.mlp.experts.103.down_proj", "model.layers.21.mlp.experts.104.down_proj", "model.layers.21.mlp.experts.105.down_proj", "model.layers.21.mlp.experts.106.down_proj", "model.layers.21.mlp.experts.107.down_proj", "model.layers.21.mlp.experts.108.down_proj", "model.layers.21.mlp.experts.109.down_proj", "model.layers.21.mlp.experts.110.down_proj", "model.layers.21.mlp.experts.111.down_proj", "model.layers.21.mlp.experts.112.down_proj", "model.layers.21.mlp.experts.113.down_proj", "model.layers.21.mlp.experts.114.down_proj", "model.layers.21.mlp.experts.115.down_proj", "model.layers.21.mlp.experts.116.down_proj", "model.layers.21.mlp.experts.117.down_proj", "model.layers.21.mlp.experts.118.down_proj", "model.layers.21.mlp.experts.119.down_proj", "model.layers.21.mlp.experts.120.down_proj", "model.layers.21.mlp.experts.121.down_proj", "model.layers.21.mlp.experts.122.down_proj", "model.layers.21.mlp.experts.123.down_proj", "model.layers.21.mlp.experts.124.down_proj", "model.layers.21.mlp.experts.125.down_proj", "model.layers.21.mlp.experts.126.down_proj", "model.layers.21.mlp.experts.127.down_proj", "model.layers.21.mlp.experts.128.down_proj", "model.layers.21.mlp.experts.129.down_proj", "model.layers.21.mlp.experts.130.down_proj", "model.layers.21.mlp.experts.131.down_proj", "model.layers.21.mlp.experts.132.down_proj", "model.layers.21.mlp.experts.133.down_proj", "model.layers.21.mlp.experts.134.down_proj", "model.layers.21.mlp.experts.135.down_proj", "model.layers.21.mlp.experts.136.down_proj", "model.layers.21.mlp.experts.137.down_proj", "model.layers.21.mlp.experts.138.down_proj", "model.layers.21.mlp.experts.139.down_proj", "model.layers.21.mlp.experts.140.down_proj", "model.layers.21.mlp.experts.141.down_proj", "model.layers.21.mlp.experts.142.down_proj", "model.layers.21.mlp.experts.143.down_proj", "model.layers.21.mlp.experts.144.down_proj", "model.layers.21.mlp.experts.145.down_proj", "model.layers.21.mlp.experts.146.down_proj", "model.layers.21.mlp.experts.147.down_proj", "model.layers.21.mlp.experts.148.down_proj", "model.layers.21.mlp.experts.149.down_proj", "model.layers.21.mlp.experts.150.down_proj", "model.layers.21.mlp.experts.151.down_proj", "model.layers.21.mlp.experts.152.down_proj", "model.layers.21.mlp.experts.153.down_proj", "model.layers.21.mlp.experts.154.down_proj", "model.layers.21.mlp.experts.155.down_proj", "model.layers.21.mlp.experts.156.down_proj", "model.layers.21.mlp.experts.157.down_proj", "model.layers.21.mlp.experts.158.down_proj", "model.layers.21.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.5826255548745394e-05, "dbits": 2516582400 } ] }, { "idx": 129, "layers": [ "model.layers.22.self_attn.q_proj" ], "candidates": [ { "dkld": -1.7641286831349134e-06, "dbits": 125829120 } ] }, { "idx": 130, "layers": [ "model.layers.22.self_attn.k_proj", "model.layers.22.self_attn.v_proj" ], "candidates": [ { "dkld": -8.95543198566886e-06, "dbits": 20971520 } ] }, { "idx": 131, "layers": [ "model.layers.22.self_attn.o_proj" ], "candidates": [ { "dkld": -1.4509714674204588e-06, "dbits": 125829120 } ] }, { "idx": 132, "layers": [ "model.layers.22.mlp.shared_experts.gate_proj", "model.layers.22.mlp.shared_experts.up_proj", "model.layers.22.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.570715827867432e-06, "dbits": 47185920 } ] }, { "idx": 133, "layers": [ "model.layers.22.mlp.experts.0.gate_proj", "model.layers.22.mlp.experts.1.gate_proj", "model.layers.22.mlp.experts.2.gate_proj", "model.layers.22.mlp.experts.3.gate_proj", "model.layers.22.mlp.experts.4.gate_proj", "model.layers.22.mlp.experts.5.gate_proj", "model.layers.22.mlp.experts.6.gate_proj", "model.layers.22.mlp.experts.7.gate_proj", "model.layers.22.mlp.experts.8.gate_proj", "model.layers.22.mlp.experts.9.gate_proj", "model.layers.22.mlp.experts.10.gate_proj", "model.layers.22.mlp.experts.11.gate_proj", "model.layers.22.mlp.experts.12.gate_proj", "model.layers.22.mlp.experts.13.gate_proj", "model.layers.22.mlp.experts.14.gate_proj", "model.layers.22.mlp.experts.15.gate_proj", "model.layers.22.mlp.experts.16.gate_proj", "model.layers.22.mlp.experts.17.gate_proj", "model.layers.22.mlp.experts.18.gate_proj", "model.layers.22.mlp.experts.19.gate_proj", "model.layers.22.mlp.experts.20.gate_proj", "model.layers.22.mlp.experts.21.gate_proj", "model.layers.22.mlp.experts.22.gate_proj", "model.layers.22.mlp.experts.23.gate_proj", "model.layers.22.mlp.experts.24.gate_proj", "model.layers.22.mlp.experts.25.gate_proj", "model.layers.22.mlp.experts.26.gate_proj", "model.layers.22.mlp.experts.27.gate_proj", "model.layers.22.mlp.experts.28.gate_proj", "model.layers.22.mlp.experts.29.gate_proj", "model.layers.22.mlp.experts.30.gate_proj", "model.layers.22.mlp.experts.31.gate_proj", "model.layers.22.mlp.experts.32.gate_proj", "model.layers.22.mlp.experts.33.gate_proj", "model.layers.22.mlp.experts.34.gate_proj", "model.layers.22.mlp.experts.35.gate_proj", "model.layers.22.mlp.experts.36.gate_proj", "model.layers.22.mlp.experts.37.gate_proj", "model.layers.22.mlp.experts.38.gate_proj", "model.layers.22.mlp.experts.39.gate_proj", "model.layers.22.mlp.experts.40.gate_proj", "model.layers.22.mlp.experts.41.gate_proj", "model.layers.22.mlp.experts.42.gate_proj", "model.layers.22.mlp.experts.43.gate_proj", "model.layers.22.mlp.experts.44.gate_proj", "model.layers.22.mlp.experts.45.gate_proj", "model.layers.22.mlp.experts.46.gate_proj", "model.layers.22.mlp.experts.47.gate_proj", "model.layers.22.mlp.experts.48.gate_proj", "model.layers.22.mlp.experts.49.gate_proj", "model.layers.22.mlp.experts.50.gate_proj", "model.layers.22.mlp.experts.51.gate_proj", "model.layers.22.mlp.experts.52.gate_proj", "model.layers.22.mlp.experts.53.gate_proj", "model.layers.22.mlp.experts.54.gate_proj", "model.layers.22.mlp.experts.55.gate_proj", "model.layers.22.mlp.experts.56.gate_proj", "model.layers.22.mlp.experts.57.gate_proj", "model.layers.22.mlp.experts.58.gate_proj", "model.layers.22.mlp.experts.59.gate_proj", "model.layers.22.mlp.experts.60.gate_proj", "model.layers.22.mlp.experts.61.gate_proj", "model.layers.22.mlp.experts.62.gate_proj", "model.layers.22.mlp.experts.63.gate_proj", "model.layers.22.mlp.experts.64.gate_proj", "model.layers.22.mlp.experts.65.gate_proj", "model.layers.22.mlp.experts.66.gate_proj", "model.layers.22.mlp.experts.67.gate_proj", "model.layers.22.mlp.experts.68.gate_proj", "model.layers.22.mlp.experts.69.gate_proj", "model.layers.22.mlp.experts.70.gate_proj", "model.layers.22.mlp.experts.71.gate_proj", "model.layers.22.mlp.experts.72.gate_proj", "model.layers.22.mlp.experts.73.gate_proj", "model.layers.22.mlp.experts.74.gate_proj", "model.layers.22.mlp.experts.75.gate_proj", "model.layers.22.mlp.experts.76.gate_proj", "model.layers.22.mlp.experts.77.gate_proj", "model.layers.22.mlp.experts.78.gate_proj", "model.layers.22.mlp.experts.79.gate_proj", "model.layers.22.mlp.experts.80.gate_proj", "model.layers.22.mlp.experts.81.gate_proj", "model.layers.22.mlp.experts.82.gate_proj", "model.layers.22.mlp.experts.83.gate_proj", "model.layers.22.mlp.experts.84.gate_proj", "model.layers.22.mlp.experts.85.gate_proj", "model.layers.22.mlp.experts.86.gate_proj", "model.layers.22.mlp.experts.87.gate_proj", "model.layers.22.mlp.experts.88.gate_proj", "model.layers.22.mlp.experts.89.gate_proj", "model.layers.22.mlp.experts.90.gate_proj", "model.layers.22.mlp.experts.91.gate_proj", "model.layers.22.mlp.experts.92.gate_proj", "model.layers.22.mlp.experts.93.gate_proj", "model.layers.22.mlp.experts.94.gate_proj", "model.layers.22.mlp.experts.95.gate_proj", "model.layers.22.mlp.experts.96.gate_proj", "model.layers.22.mlp.experts.97.gate_proj", "model.layers.22.mlp.experts.98.gate_proj", "model.layers.22.mlp.experts.99.gate_proj", "model.layers.22.mlp.experts.100.gate_proj", "model.layers.22.mlp.experts.101.gate_proj", "model.layers.22.mlp.experts.102.gate_proj", "model.layers.22.mlp.experts.103.gate_proj", "model.layers.22.mlp.experts.104.gate_proj", "model.layers.22.mlp.experts.105.gate_proj", "model.layers.22.mlp.experts.106.gate_proj", "model.layers.22.mlp.experts.107.gate_proj", "model.layers.22.mlp.experts.108.gate_proj", "model.layers.22.mlp.experts.109.gate_proj", "model.layers.22.mlp.experts.110.gate_proj", "model.layers.22.mlp.experts.111.gate_proj", "model.layers.22.mlp.experts.112.gate_proj", "model.layers.22.mlp.experts.113.gate_proj", "model.layers.22.mlp.experts.114.gate_proj", "model.layers.22.mlp.experts.115.gate_proj", "model.layers.22.mlp.experts.116.gate_proj", "model.layers.22.mlp.experts.117.gate_proj", "model.layers.22.mlp.experts.118.gate_proj", "model.layers.22.mlp.experts.119.gate_proj", "model.layers.22.mlp.experts.120.gate_proj", "model.layers.22.mlp.experts.121.gate_proj", "model.layers.22.mlp.experts.122.gate_proj", "model.layers.22.mlp.experts.123.gate_proj", "model.layers.22.mlp.experts.124.gate_proj", "model.layers.22.mlp.experts.125.gate_proj", "model.layers.22.mlp.experts.126.gate_proj", "model.layers.22.mlp.experts.127.gate_proj", "model.layers.22.mlp.experts.128.gate_proj", "model.layers.22.mlp.experts.129.gate_proj", "model.layers.22.mlp.experts.130.gate_proj", "model.layers.22.mlp.experts.131.gate_proj", "model.layers.22.mlp.experts.132.gate_proj", "model.layers.22.mlp.experts.133.gate_proj", "model.layers.22.mlp.experts.134.gate_proj", "model.layers.22.mlp.experts.135.gate_proj", "model.layers.22.mlp.experts.136.gate_proj", "model.layers.22.mlp.experts.137.gate_proj", "model.layers.22.mlp.experts.138.gate_proj", "model.layers.22.mlp.experts.139.gate_proj", "model.layers.22.mlp.experts.140.gate_proj", "model.layers.22.mlp.experts.141.gate_proj", "model.layers.22.mlp.experts.142.gate_proj", "model.layers.22.mlp.experts.143.gate_proj", "model.layers.22.mlp.experts.144.gate_proj", "model.layers.22.mlp.experts.145.gate_proj", "model.layers.22.mlp.experts.146.gate_proj", "model.layers.22.mlp.experts.147.gate_proj", "model.layers.22.mlp.experts.148.gate_proj", "model.layers.22.mlp.experts.149.gate_proj", "model.layers.22.mlp.experts.150.gate_proj", "model.layers.22.mlp.experts.151.gate_proj", "model.layers.22.mlp.experts.152.gate_proj", "model.layers.22.mlp.experts.153.gate_proj", "model.layers.22.mlp.experts.154.gate_proj", "model.layers.22.mlp.experts.155.gate_proj", "model.layers.22.mlp.experts.156.gate_proj", "model.layers.22.mlp.experts.157.gate_proj", "model.layers.22.mlp.experts.158.gate_proj", "model.layers.22.mlp.experts.159.gate_proj", "model.layers.22.mlp.experts.0.up_proj", "model.layers.22.mlp.experts.1.up_proj", "model.layers.22.mlp.experts.2.up_proj", "model.layers.22.mlp.experts.3.up_proj", "model.layers.22.mlp.experts.4.up_proj", "model.layers.22.mlp.experts.5.up_proj", "model.layers.22.mlp.experts.6.up_proj", "model.layers.22.mlp.experts.7.up_proj", "model.layers.22.mlp.experts.8.up_proj", "model.layers.22.mlp.experts.9.up_proj", "model.layers.22.mlp.experts.10.up_proj", "model.layers.22.mlp.experts.11.up_proj", "model.layers.22.mlp.experts.12.up_proj", "model.layers.22.mlp.experts.13.up_proj", "model.layers.22.mlp.experts.14.up_proj", "model.layers.22.mlp.experts.15.up_proj", "model.layers.22.mlp.experts.16.up_proj", "model.layers.22.mlp.experts.17.up_proj", "model.layers.22.mlp.experts.18.up_proj", "model.layers.22.mlp.experts.19.up_proj", "model.layers.22.mlp.experts.20.up_proj", "model.layers.22.mlp.experts.21.up_proj", "model.layers.22.mlp.experts.22.up_proj", "model.layers.22.mlp.experts.23.up_proj", "model.layers.22.mlp.experts.24.up_proj", "model.layers.22.mlp.experts.25.up_proj", "model.layers.22.mlp.experts.26.up_proj", "model.layers.22.mlp.experts.27.up_proj", "model.layers.22.mlp.experts.28.up_proj", "model.layers.22.mlp.experts.29.up_proj", "model.layers.22.mlp.experts.30.up_proj", "model.layers.22.mlp.experts.31.up_proj", "model.layers.22.mlp.experts.32.up_proj", "model.layers.22.mlp.experts.33.up_proj", "model.layers.22.mlp.experts.34.up_proj", "model.layers.22.mlp.experts.35.up_proj", "model.layers.22.mlp.experts.36.up_proj", "model.layers.22.mlp.experts.37.up_proj", "model.layers.22.mlp.experts.38.up_proj", "model.layers.22.mlp.experts.39.up_proj", "model.layers.22.mlp.experts.40.up_proj", "model.layers.22.mlp.experts.41.up_proj", "model.layers.22.mlp.experts.42.up_proj", "model.layers.22.mlp.experts.43.up_proj", "model.layers.22.mlp.experts.44.up_proj", "model.layers.22.mlp.experts.45.up_proj", "model.layers.22.mlp.experts.46.up_proj", "model.layers.22.mlp.experts.47.up_proj", "model.layers.22.mlp.experts.48.up_proj", "model.layers.22.mlp.experts.49.up_proj", "model.layers.22.mlp.experts.50.up_proj", "model.layers.22.mlp.experts.51.up_proj", "model.layers.22.mlp.experts.52.up_proj", "model.layers.22.mlp.experts.53.up_proj", "model.layers.22.mlp.experts.54.up_proj", "model.layers.22.mlp.experts.55.up_proj", "model.layers.22.mlp.experts.56.up_proj", "model.layers.22.mlp.experts.57.up_proj", "model.layers.22.mlp.experts.58.up_proj", "model.layers.22.mlp.experts.59.up_proj", "model.layers.22.mlp.experts.60.up_proj", "model.layers.22.mlp.experts.61.up_proj", "model.layers.22.mlp.experts.62.up_proj", "model.layers.22.mlp.experts.63.up_proj", "model.layers.22.mlp.experts.64.up_proj", "model.layers.22.mlp.experts.65.up_proj", "model.layers.22.mlp.experts.66.up_proj", "model.layers.22.mlp.experts.67.up_proj", "model.layers.22.mlp.experts.68.up_proj", "model.layers.22.mlp.experts.69.up_proj", "model.layers.22.mlp.experts.70.up_proj", "model.layers.22.mlp.experts.71.up_proj", "model.layers.22.mlp.experts.72.up_proj", "model.layers.22.mlp.experts.73.up_proj", "model.layers.22.mlp.experts.74.up_proj", "model.layers.22.mlp.experts.75.up_proj", "model.layers.22.mlp.experts.76.up_proj", "model.layers.22.mlp.experts.77.up_proj", "model.layers.22.mlp.experts.78.up_proj", "model.layers.22.mlp.experts.79.up_proj", "model.layers.22.mlp.experts.80.up_proj", "model.layers.22.mlp.experts.81.up_proj", "model.layers.22.mlp.experts.82.up_proj", "model.layers.22.mlp.experts.83.up_proj", "model.layers.22.mlp.experts.84.up_proj", "model.layers.22.mlp.experts.85.up_proj", "model.layers.22.mlp.experts.86.up_proj", "model.layers.22.mlp.experts.87.up_proj", "model.layers.22.mlp.experts.88.up_proj", "model.layers.22.mlp.experts.89.up_proj", "model.layers.22.mlp.experts.90.up_proj", "model.layers.22.mlp.experts.91.up_proj", "model.layers.22.mlp.experts.92.up_proj", "model.layers.22.mlp.experts.93.up_proj", "model.layers.22.mlp.experts.94.up_proj", "model.layers.22.mlp.experts.95.up_proj", "model.layers.22.mlp.experts.96.up_proj", "model.layers.22.mlp.experts.97.up_proj", "model.layers.22.mlp.experts.98.up_proj", "model.layers.22.mlp.experts.99.up_proj", "model.layers.22.mlp.experts.100.up_proj", "model.layers.22.mlp.experts.101.up_proj", "model.layers.22.mlp.experts.102.up_proj", "model.layers.22.mlp.experts.103.up_proj", "model.layers.22.mlp.experts.104.up_proj", "model.layers.22.mlp.experts.105.up_proj", "model.layers.22.mlp.experts.106.up_proj", "model.layers.22.mlp.experts.107.up_proj", "model.layers.22.mlp.experts.108.up_proj", "model.layers.22.mlp.experts.109.up_proj", "model.layers.22.mlp.experts.110.up_proj", "model.layers.22.mlp.experts.111.up_proj", "model.layers.22.mlp.experts.112.up_proj", "model.layers.22.mlp.experts.113.up_proj", "model.layers.22.mlp.experts.114.up_proj", "model.layers.22.mlp.experts.115.up_proj", "model.layers.22.mlp.experts.116.up_proj", "model.layers.22.mlp.experts.117.up_proj", "model.layers.22.mlp.experts.118.up_proj", "model.layers.22.mlp.experts.119.up_proj", "model.layers.22.mlp.experts.120.up_proj", "model.layers.22.mlp.experts.121.up_proj", "model.layers.22.mlp.experts.122.up_proj", "model.layers.22.mlp.experts.123.up_proj", "model.layers.22.mlp.experts.124.up_proj", "model.layers.22.mlp.experts.125.up_proj", "model.layers.22.mlp.experts.126.up_proj", "model.layers.22.mlp.experts.127.up_proj", "model.layers.22.mlp.experts.128.up_proj", "model.layers.22.mlp.experts.129.up_proj", "model.layers.22.mlp.experts.130.up_proj", "model.layers.22.mlp.experts.131.up_proj", "model.layers.22.mlp.experts.132.up_proj", "model.layers.22.mlp.experts.133.up_proj", "model.layers.22.mlp.experts.134.up_proj", "model.layers.22.mlp.experts.135.up_proj", "model.layers.22.mlp.experts.136.up_proj", "model.layers.22.mlp.experts.137.up_proj", "model.layers.22.mlp.experts.138.up_proj", "model.layers.22.mlp.experts.139.up_proj", "model.layers.22.mlp.experts.140.up_proj", "model.layers.22.mlp.experts.141.up_proj", "model.layers.22.mlp.experts.142.up_proj", "model.layers.22.mlp.experts.143.up_proj", "model.layers.22.mlp.experts.144.up_proj", "model.layers.22.mlp.experts.145.up_proj", "model.layers.22.mlp.experts.146.up_proj", "model.layers.22.mlp.experts.147.up_proj", "model.layers.22.mlp.experts.148.up_proj", "model.layers.22.mlp.experts.149.up_proj", "model.layers.22.mlp.experts.150.up_proj", "model.layers.22.mlp.experts.151.up_proj", "model.layers.22.mlp.experts.152.up_proj", "model.layers.22.mlp.experts.153.up_proj", "model.layers.22.mlp.experts.154.up_proj", "model.layers.22.mlp.experts.155.up_proj", "model.layers.22.mlp.experts.156.up_proj", "model.layers.22.mlp.experts.157.up_proj", "model.layers.22.mlp.experts.158.up_proj", "model.layers.22.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.576220478862524e-06, "dbits": 5033164800 } ] }, { "idx": 134, "layers": [ "model.layers.22.mlp.experts.0.down_proj", "model.layers.22.mlp.experts.1.down_proj", "model.layers.22.mlp.experts.2.down_proj", "model.layers.22.mlp.experts.3.down_proj", "model.layers.22.mlp.experts.4.down_proj", "model.layers.22.mlp.experts.5.down_proj", "model.layers.22.mlp.experts.6.down_proj", "model.layers.22.mlp.experts.7.down_proj", "model.layers.22.mlp.experts.8.down_proj", "model.layers.22.mlp.experts.9.down_proj", "model.layers.22.mlp.experts.10.down_proj", "model.layers.22.mlp.experts.11.down_proj", "model.layers.22.mlp.experts.12.down_proj", "model.layers.22.mlp.experts.13.down_proj", "model.layers.22.mlp.experts.14.down_proj", "model.layers.22.mlp.experts.15.down_proj", "model.layers.22.mlp.experts.16.down_proj", "model.layers.22.mlp.experts.17.down_proj", "model.layers.22.mlp.experts.18.down_proj", "model.layers.22.mlp.experts.19.down_proj", "model.layers.22.mlp.experts.20.down_proj", "model.layers.22.mlp.experts.21.down_proj", "model.layers.22.mlp.experts.22.down_proj", "model.layers.22.mlp.experts.23.down_proj", "model.layers.22.mlp.experts.24.down_proj", "model.layers.22.mlp.experts.25.down_proj", "model.layers.22.mlp.experts.26.down_proj", "model.layers.22.mlp.experts.27.down_proj", "model.layers.22.mlp.experts.28.down_proj", "model.layers.22.mlp.experts.29.down_proj", "model.layers.22.mlp.experts.30.down_proj", "model.layers.22.mlp.experts.31.down_proj", "model.layers.22.mlp.experts.32.down_proj", "model.layers.22.mlp.experts.33.down_proj", "model.layers.22.mlp.experts.34.down_proj", "model.layers.22.mlp.experts.35.down_proj", "model.layers.22.mlp.experts.36.down_proj", "model.layers.22.mlp.experts.37.down_proj", "model.layers.22.mlp.experts.38.down_proj", "model.layers.22.mlp.experts.39.down_proj", "model.layers.22.mlp.experts.40.down_proj", "model.layers.22.mlp.experts.41.down_proj", "model.layers.22.mlp.experts.42.down_proj", "model.layers.22.mlp.experts.43.down_proj", "model.layers.22.mlp.experts.44.down_proj", "model.layers.22.mlp.experts.45.down_proj", "model.layers.22.mlp.experts.46.down_proj", "model.layers.22.mlp.experts.47.down_proj", "model.layers.22.mlp.experts.48.down_proj", "model.layers.22.mlp.experts.49.down_proj", "model.layers.22.mlp.experts.50.down_proj", "model.layers.22.mlp.experts.51.down_proj", "model.layers.22.mlp.experts.52.down_proj", "model.layers.22.mlp.experts.53.down_proj", "model.layers.22.mlp.experts.54.down_proj", "model.layers.22.mlp.experts.55.down_proj", "model.layers.22.mlp.experts.56.down_proj", "model.layers.22.mlp.experts.57.down_proj", "model.layers.22.mlp.experts.58.down_proj", "model.layers.22.mlp.experts.59.down_proj", "model.layers.22.mlp.experts.60.down_proj", "model.layers.22.mlp.experts.61.down_proj", "model.layers.22.mlp.experts.62.down_proj", "model.layers.22.mlp.experts.63.down_proj", "model.layers.22.mlp.experts.64.down_proj", "model.layers.22.mlp.experts.65.down_proj", "model.layers.22.mlp.experts.66.down_proj", "model.layers.22.mlp.experts.67.down_proj", "model.layers.22.mlp.experts.68.down_proj", "model.layers.22.mlp.experts.69.down_proj", "model.layers.22.mlp.experts.70.down_proj", "model.layers.22.mlp.experts.71.down_proj", "model.layers.22.mlp.experts.72.down_proj", "model.layers.22.mlp.experts.73.down_proj", "model.layers.22.mlp.experts.74.down_proj", "model.layers.22.mlp.experts.75.down_proj", "model.layers.22.mlp.experts.76.down_proj", "model.layers.22.mlp.experts.77.down_proj", "model.layers.22.mlp.experts.78.down_proj", "model.layers.22.mlp.experts.79.down_proj", "model.layers.22.mlp.experts.80.down_proj", "model.layers.22.mlp.experts.81.down_proj", "model.layers.22.mlp.experts.82.down_proj", "model.layers.22.mlp.experts.83.down_proj", "model.layers.22.mlp.experts.84.down_proj", "model.layers.22.mlp.experts.85.down_proj", "model.layers.22.mlp.experts.86.down_proj", "model.layers.22.mlp.experts.87.down_proj", "model.layers.22.mlp.experts.88.down_proj", "model.layers.22.mlp.experts.89.down_proj", "model.layers.22.mlp.experts.90.down_proj", "model.layers.22.mlp.experts.91.down_proj", "model.layers.22.mlp.experts.92.down_proj", "model.layers.22.mlp.experts.93.down_proj", "model.layers.22.mlp.experts.94.down_proj", "model.layers.22.mlp.experts.95.down_proj", "model.layers.22.mlp.experts.96.down_proj", "model.layers.22.mlp.experts.97.down_proj", "model.layers.22.mlp.experts.98.down_proj", "model.layers.22.mlp.experts.99.down_proj", "model.layers.22.mlp.experts.100.down_proj", "model.layers.22.mlp.experts.101.down_proj", "model.layers.22.mlp.experts.102.down_proj", "model.layers.22.mlp.experts.103.down_proj", "model.layers.22.mlp.experts.104.down_proj", "model.layers.22.mlp.experts.105.down_proj", "model.layers.22.mlp.experts.106.down_proj", "model.layers.22.mlp.experts.107.down_proj", "model.layers.22.mlp.experts.108.down_proj", "model.layers.22.mlp.experts.109.down_proj", "model.layers.22.mlp.experts.110.down_proj", "model.layers.22.mlp.experts.111.down_proj", "model.layers.22.mlp.experts.112.down_proj", "model.layers.22.mlp.experts.113.down_proj", "model.layers.22.mlp.experts.114.down_proj", "model.layers.22.mlp.experts.115.down_proj", "model.layers.22.mlp.experts.116.down_proj", "model.layers.22.mlp.experts.117.down_proj", "model.layers.22.mlp.experts.118.down_proj", "model.layers.22.mlp.experts.119.down_proj", "model.layers.22.mlp.experts.120.down_proj", "model.layers.22.mlp.experts.121.down_proj", "model.layers.22.mlp.experts.122.down_proj", "model.layers.22.mlp.experts.123.down_proj", "model.layers.22.mlp.experts.124.down_proj", "model.layers.22.mlp.experts.125.down_proj", "model.layers.22.mlp.experts.126.down_proj", "model.layers.22.mlp.experts.127.down_proj", "model.layers.22.mlp.experts.128.down_proj", "model.layers.22.mlp.experts.129.down_proj", "model.layers.22.mlp.experts.130.down_proj", "model.layers.22.mlp.experts.131.down_proj", "model.layers.22.mlp.experts.132.down_proj", "model.layers.22.mlp.experts.133.down_proj", "model.layers.22.mlp.experts.134.down_proj", "model.layers.22.mlp.experts.135.down_proj", "model.layers.22.mlp.experts.136.down_proj", "model.layers.22.mlp.experts.137.down_proj", "model.layers.22.mlp.experts.138.down_proj", "model.layers.22.mlp.experts.139.down_proj", "model.layers.22.mlp.experts.140.down_proj", "model.layers.22.mlp.experts.141.down_proj", "model.layers.22.mlp.experts.142.down_proj", "model.layers.22.mlp.experts.143.down_proj", "model.layers.22.mlp.experts.144.down_proj", "model.layers.22.mlp.experts.145.down_proj", "model.layers.22.mlp.experts.146.down_proj", "model.layers.22.mlp.experts.147.down_proj", "model.layers.22.mlp.experts.148.down_proj", "model.layers.22.mlp.experts.149.down_proj", "model.layers.22.mlp.experts.150.down_proj", "model.layers.22.mlp.experts.151.down_proj", "model.layers.22.mlp.experts.152.down_proj", "model.layers.22.mlp.experts.153.down_proj", "model.layers.22.mlp.experts.154.down_proj", "model.layers.22.mlp.experts.155.down_proj", "model.layers.22.mlp.experts.156.down_proj", "model.layers.22.mlp.experts.157.down_proj", "model.layers.22.mlp.experts.158.down_proj", "model.layers.22.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 9.031471563503146e-06, "dbits": 2516582400 } ] }, { "idx": 135, "layers": [ "model.layers.23.self_attn.q_proj" ], "candidates": [ { "dkld": 1.6656046500429468e-05, "dbits": 125829120 } ] }, { "idx": 136, "layers": [ "model.layers.23.self_attn.k_proj", "model.layers.23.self_attn.v_proj" ], "candidates": [ { "dkld": -2.7736861375160537e-05, "dbits": 20971520 } ] }, { "idx": 137, "layers": [ "model.layers.23.self_attn.o_proj" ], "candidates": [ { "dkld": -1.0859279427677393e-05, "dbits": 125829120 } ] }, { "idx": 138, "layers": [ "model.layers.23.mlp.shared_experts.gate_proj", "model.layers.23.mlp.shared_experts.up_proj", "model.layers.23.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.552770406007767e-06, "dbits": 47185920 } ] }, { "idx": 139, "layers": [ "model.layers.23.mlp.experts.0.gate_proj", "model.layers.23.mlp.experts.1.gate_proj", "model.layers.23.mlp.experts.2.gate_proj", "model.layers.23.mlp.experts.3.gate_proj", "model.layers.23.mlp.experts.4.gate_proj", "model.layers.23.mlp.experts.5.gate_proj", "model.layers.23.mlp.experts.6.gate_proj", "model.layers.23.mlp.experts.7.gate_proj", "model.layers.23.mlp.experts.8.gate_proj", "model.layers.23.mlp.experts.9.gate_proj", "model.layers.23.mlp.experts.10.gate_proj", "model.layers.23.mlp.experts.11.gate_proj", "model.layers.23.mlp.experts.12.gate_proj", "model.layers.23.mlp.experts.13.gate_proj", "model.layers.23.mlp.experts.14.gate_proj", "model.layers.23.mlp.experts.15.gate_proj", "model.layers.23.mlp.experts.16.gate_proj", "model.layers.23.mlp.experts.17.gate_proj", "model.layers.23.mlp.experts.18.gate_proj", "model.layers.23.mlp.experts.19.gate_proj", "model.layers.23.mlp.experts.20.gate_proj", "model.layers.23.mlp.experts.21.gate_proj", "model.layers.23.mlp.experts.22.gate_proj", "model.layers.23.mlp.experts.23.gate_proj", "model.layers.23.mlp.experts.24.gate_proj", "model.layers.23.mlp.experts.25.gate_proj", "model.layers.23.mlp.experts.26.gate_proj", "model.layers.23.mlp.experts.27.gate_proj", "model.layers.23.mlp.experts.28.gate_proj", "model.layers.23.mlp.experts.29.gate_proj", "model.layers.23.mlp.experts.30.gate_proj", "model.layers.23.mlp.experts.31.gate_proj", "model.layers.23.mlp.experts.32.gate_proj", "model.layers.23.mlp.experts.33.gate_proj", "model.layers.23.mlp.experts.34.gate_proj", "model.layers.23.mlp.experts.35.gate_proj", "model.layers.23.mlp.experts.36.gate_proj", "model.layers.23.mlp.experts.37.gate_proj", "model.layers.23.mlp.experts.38.gate_proj", "model.layers.23.mlp.experts.39.gate_proj", "model.layers.23.mlp.experts.40.gate_proj", "model.layers.23.mlp.experts.41.gate_proj", "model.layers.23.mlp.experts.42.gate_proj", "model.layers.23.mlp.experts.43.gate_proj", "model.layers.23.mlp.experts.44.gate_proj", "model.layers.23.mlp.experts.45.gate_proj", "model.layers.23.mlp.experts.46.gate_proj", "model.layers.23.mlp.experts.47.gate_proj", "model.layers.23.mlp.experts.48.gate_proj", "model.layers.23.mlp.experts.49.gate_proj", "model.layers.23.mlp.experts.50.gate_proj", "model.layers.23.mlp.experts.51.gate_proj", "model.layers.23.mlp.experts.52.gate_proj", "model.layers.23.mlp.experts.53.gate_proj", "model.layers.23.mlp.experts.54.gate_proj", "model.layers.23.mlp.experts.55.gate_proj", "model.layers.23.mlp.experts.56.gate_proj", "model.layers.23.mlp.experts.57.gate_proj", "model.layers.23.mlp.experts.58.gate_proj", "model.layers.23.mlp.experts.59.gate_proj", "model.layers.23.mlp.experts.60.gate_proj", "model.layers.23.mlp.experts.61.gate_proj", "model.layers.23.mlp.experts.62.gate_proj", "model.layers.23.mlp.experts.63.gate_proj", "model.layers.23.mlp.experts.64.gate_proj", "model.layers.23.mlp.experts.65.gate_proj", "model.layers.23.mlp.experts.66.gate_proj", "model.layers.23.mlp.experts.67.gate_proj", "model.layers.23.mlp.experts.68.gate_proj", "model.layers.23.mlp.experts.69.gate_proj", "model.layers.23.mlp.experts.70.gate_proj", "model.layers.23.mlp.experts.71.gate_proj", "model.layers.23.mlp.experts.72.gate_proj", "model.layers.23.mlp.experts.73.gate_proj", "model.layers.23.mlp.experts.74.gate_proj", "model.layers.23.mlp.experts.75.gate_proj", "model.layers.23.mlp.experts.76.gate_proj", "model.layers.23.mlp.experts.77.gate_proj", "model.layers.23.mlp.experts.78.gate_proj", "model.layers.23.mlp.experts.79.gate_proj", "model.layers.23.mlp.experts.80.gate_proj", "model.layers.23.mlp.experts.81.gate_proj", "model.layers.23.mlp.experts.82.gate_proj", "model.layers.23.mlp.experts.83.gate_proj", "model.layers.23.mlp.experts.84.gate_proj", "model.layers.23.mlp.experts.85.gate_proj", "model.layers.23.mlp.experts.86.gate_proj", "model.layers.23.mlp.experts.87.gate_proj", "model.layers.23.mlp.experts.88.gate_proj", "model.layers.23.mlp.experts.89.gate_proj", "model.layers.23.mlp.experts.90.gate_proj", "model.layers.23.mlp.experts.91.gate_proj", "model.layers.23.mlp.experts.92.gate_proj", "model.layers.23.mlp.experts.93.gate_proj", "model.layers.23.mlp.experts.94.gate_proj", "model.layers.23.mlp.experts.95.gate_proj", "model.layers.23.mlp.experts.96.gate_proj", "model.layers.23.mlp.experts.97.gate_proj", "model.layers.23.mlp.experts.98.gate_proj", "model.layers.23.mlp.experts.99.gate_proj", "model.layers.23.mlp.experts.100.gate_proj", "model.layers.23.mlp.experts.101.gate_proj", "model.layers.23.mlp.experts.102.gate_proj", "model.layers.23.mlp.experts.103.gate_proj", "model.layers.23.mlp.experts.104.gate_proj", "model.layers.23.mlp.experts.105.gate_proj", "model.layers.23.mlp.experts.106.gate_proj", "model.layers.23.mlp.experts.107.gate_proj", "model.layers.23.mlp.experts.108.gate_proj", "model.layers.23.mlp.experts.109.gate_proj", "model.layers.23.mlp.experts.110.gate_proj", "model.layers.23.mlp.experts.111.gate_proj", "model.layers.23.mlp.experts.112.gate_proj", "model.layers.23.mlp.experts.113.gate_proj", "model.layers.23.mlp.experts.114.gate_proj", "model.layers.23.mlp.experts.115.gate_proj", "model.layers.23.mlp.experts.116.gate_proj", "model.layers.23.mlp.experts.117.gate_proj", "model.layers.23.mlp.experts.118.gate_proj", "model.layers.23.mlp.experts.119.gate_proj", "model.layers.23.mlp.experts.120.gate_proj", "model.layers.23.mlp.experts.121.gate_proj", "model.layers.23.mlp.experts.122.gate_proj", "model.layers.23.mlp.experts.123.gate_proj", "model.layers.23.mlp.experts.124.gate_proj", "model.layers.23.mlp.experts.125.gate_proj", "model.layers.23.mlp.experts.126.gate_proj", "model.layers.23.mlp.experts.127.gate_proj", "model.layers.23.mlp.experts.128.gate_proj", "model.layers.23.mlp.experts.129.gate_proj", "model.layers.23.mlp.experts.130.gate_proj", "model.layers.23.mlp.experts.131.gate_proj", "model.layers.23.mlp.experts.132.gate_proj", "model.layers.23.mlp.experts.133.gate_proj", "model.layers.23.mlp.experts.134.gate_proj", "model.layers.23.mlp.experts.135.gate_proj", "model.layers.23.mlp.experts.136.gate_proj", "model.layers.23.mlp.experts.137.gate_proj", "model.layers.23.mlp.experts.138.gate_proj", "model.layers.23.mlp.experts.139.gate_proj", "model.layers.23.mlp.experts.140.gate_proj", "model.layers.23.mlp.experts.141.gate_proj", "model.layers.23.mlp.experts.142.gate_proj", "model.layers.23.mlp.experts.143.gate_proj", "model.layers.23.mlp.experts.144.gate_proj", "model.layers.23.mlp.experts.145.gate_proj", "model.layers.23.mlp.experts.146.gate_proj", "model.layers.23.mlp.experts.147.gate_proj", "model.layers.23.mlp.experts.148.gate_proj", "model.layers.23.mlp.experts.149.gate_proj", "model.layers.23.mlp.experts.150.gate_proj", "model.layers.23.mlp.experts.151.gate_proj", "model.layers.23.mlp.experts.152.gate_proj", "model.layers.23.mlp.experts.153.gate_proj", "model.layers.23.mlp.experts.154.gate_proj", "model.layers.23.mlp.experts.155.gate_proj", "model.layers.23.mlp.experts.156.gate_proj", "model.layers.23.mlp.experts.157.gate_proj", "model.layers.23.mlp.experts.158.gate_proj", "model.layers.23.mlp.experts.159.gate_proj", "model.layers.23.mlp.experts.0.up_proj", "model.layers.23.mlp.experts.1.up_proj", "model.layers.23.mlp.experts.2.up_proj", "model.layers.23.mlp.experts.3.up_proj", "model.layers.23.mlp.experts.4.up_proj", "model.layers.23.mlp.experts.5.up_proj", "model.layers.23.mlp.experts.6.up_proj", "model.layers.23.mlp.experts.7.up_proj", "model.layers.23.mlp.experts.8.up_proj", "model.layers.23.mlp.experts.9.up_proj", "model.layers.23.mlp.experts.10.up_proj", "model.layers.23.mlp.experts.11.up_proj", "model.layers.23.mlp.experts.12.up_proj", "model.layers.23.mlp.experts.13.up_proj", "model.layers.23.mlp.experts.14.up_proj", "model.layers.23.mlp.experts.15.up_proj", "model.layers.23.mlp.experts.16.up_proj", "model.layers.23.mlp.experts.17.up_proj", "model.layers.23.mlp.experts.18.up_proj", "model.layers.23.mlp.experts.19.up_proj", "model.layers.23.mlp.experts.20.up_proj", "model.layers.23.mlp.experts.21.up_proj", "model.layers.23.mlp.experts.22.up_proj", "model.layers.23.mlp.experts.23.up_proj", "model.layers.23.mlp.experts.24.up_proj", "model.layers.23.mlp.experts.25.up_proj", "model.layers.23.mlp.experts.26.up_proj", "model.layers.23.mlp.experts.27.up_proj", "model.layers.23.mlp.experts.28.up_proj", "model.layers.23.mlp.experts.29.up_proj", "model.layers.23.mlp.experts.30.up_proj", "model.layers.23.mlp.experts.31.up_proj", "model.layers.23.mlp.experts.32.up_proj", "model.layers.23.mlp.experts.33.up_proj", "model.layers.23.mlp.experts.34.up_proj", "model.layers.23.mlp.experts.35.up_proj", "model.layers.23.mlp.experts.36.up_proj", "model.layers.23.mlp.experts.37.up_proj", "model.layers.23.mlp.experts.38.up_proj", "model.layers.23.mlp.experts.39.up_proj", "model.layers.23.mlp.experts.40.up_proj", "model.layers.23.mlp.experts.41.up_proj", "model.layers.23.mlp.experts.42.up_proj", "model.layers.23.mlp.experts.43.up_proj", "model.layers.23.mlp.experts.44.up_proj", "model.layers.23.mlp.experts.45.up_proj", "model.layers.23.mlp.experts.46.up_proj", "model.layers.23.mlp.experts.47.up_proj", "model.layers.23.mlp.experts.48.up_proj", "model.layers.23.mlp.experts.49.up_proj", "model.layers.23.mlp.experts.50.up_proj", "model.layers.23.mlp.experts.51.up_proj", "model.layers.23.mlp.experts.52.up_proj", "model.layers.23.mlp.experts.53.up_proj", "model.layers.23.mlp.experts.54.up_proj", "model.layers.23.mlp.experts.55.up_proj", "model.layers.23.mlp.experts.56.up_proj", "model.layers.23.mlp.experts.57.up_proj", "model.layers.23.mlp.experts.58.up_proj", "model.layers.23.mlp.experts.59.up_proj", "model.layers.23.mlp.experts.60.up_proj", "model.layers.23.mlp.experts.61.up_proj", "model.layers.23.mlp.experts.62.up_proj", "model.layers.23.mlp.experts.63.up_proj", "model.layers.23.mlp.experts.64.up_proj", "model.layers.23.mlp.experts.65.up_proj", "model.layers.23.mlp.experts.66.up_proj", "model.layers.23.mlp.experts.67.up_proj", "model.layers.23.mlp.experts.68.up_proj", "model.layers.23.mlp.experts.69.up_proj", "model.layers.23.mlp.experts.70.up_proj", "model.layers.23.mlp.experts.71.up_proj", "model.layers.23.mlp.experts.72.up_proj", "model.layers.23.mlp.experts.73.up_proj", "model.layers.23.mlp.experts.74.up_proj", "model.layers.23.mlp.experts.75.up_proj", "model.layers.23.mlp.experts.76.up_proj", "model.layers.23.mlp.experts.77.up_proj", "model.layers.23.mlp.experts.78.up_proj", "model.layers.23.mlp.experts.79.up_proj", "model.layers.23.mlp.experts.80.up_proj", "model.layers.23.mlp.experts.81.up_proj", "model.layers.23.mlp.experts.82.up_proj", "model.layers.23.mlp.experts.83.up_proj", "model.layers.23.mlp.experts.84.up_proj", "model.layers.23.mlp.experts.85.up_proj", "model.layers.23.mlp.experts.86.up_proj", "model.layers.23.mlp.experts.87.up_proj", "model.layers.23.mlp.experts.88.up_proj", "model.layers.23.mlp.experts.89.up_proj", "model.layers.23.mlp.experts.90.up_proj", "model.layers.23.mlp.experts.91.up_proj", "model.layers.23.mlp.experts.92.up_proj", "model.layers.23.mlp.experts.93.up_proj", "model.layers.23.mlp.experts.94.up_proj", "model.layers.23.mlp.experts.95.up_proj", "model.layers.23.mlp.experts.96.up_proj", "model.layers.23.mlp.experts.97.up_proj", "model.layers.23.mlp.experts.98.up_proj", "model.layers.23.mlp.experts.99.up_proj", "model.layers.23.mlp.experts.100.up_proj", "model.layers.23.mlp.experts.101.up_proj", "model.layers.23.mlp.experts.102.up_proj", "model.layers.23.mlp.experts.103.up_proj", "model.layers.23.mlp.experts.104.up_proj", "model.layers.23.mlp.experts.105.up_proj", "model.layers.23.mlp.experts.106.up_proj", "model.layers.23.mlp.experts.107.up_proj", "model.layers.23.mlp.experts.108.up_proj", "model.layers.23.mlp.experts.109.up_proj", "model.layers.23.mlp.experts.110.up_proj", "model.layers.23.mlp.experts.111.up_proj", "model.layers.23.mlp.experts.112.up_proj", "model.layers.23.mlp.experts.113.up_proj", "model.layers.23.mlp.experts.114.up_proj", "model.layers.23.mlp.experts.115.up_proj", "model.layers.23.mlp.experts.116.up_proj", "model.layers.23.mlp.experts.117.up_proj", "model.layers.23.mlp.experts.118.up_proj", "model.layers.23.mlp.experts.119.up_proj", "model.layers.23.mlp.experts.120.up_proj", "model.layers.23.mlp.experts.121.up_proj", "model.layers.23.mlp.experts.122.up_proj", "model.layers.23.mlp.experts.123.up_proj", "model.layers.23.mlp.experts.124.up_proj", "model.layers.23.mlp.experts.125.up_proj", "model.layers.23.mlp.experts.126.up_proj", "model.layers.23.mlp.experts.127.up_proj", "model.layers.23.mlp.experts.128.up_proj", "model.layers.23.mlp.experts.129.up_proj", "model.layers.23.mlp.experts.130.up_proj", "model.layers.23.mlp.experts.131.up_proj", "model.layers.23.mlp.experts.132.up_proj", "model.layers.23.mlp.experts.133.up_proj", "model.layers.23.mlp.experts.134.up_proj", "model.layers.23.mlp.experts.135.up_proj", "model.layers.23.mlp.experts.136.up_proj", "model.layers.23.mlp.experts.137.up_proj", "model.layers.23.mlp.experts.138.up_proj", "model.layers.23.mlp.experts.139.up_proj", "model.layers.23.mlp.experts.140.up_proj", "model.layers.23.mlp.experts.141.up_proj", "model.layers.23.mlp.experts.142.up_proj", "model.layers.23.mlp.experts.143.up_proj", "model.layers.23.mlp.experts.144.up_proj", "model.layers.23.mlp.experts.145.up_proj", "model.layers.23.mlp.experts.146.up_proj", "model.layers.23.mlp.experts.147.up_proj", "model.layers.23.mlp.experts.148.up_proj", "model.layers.23.mlp.experts.149.up_proj", "model.layers.23.mlp.experts.150.up_proj", "model.layers.23.mlp.experts.151.up_proj", "model.layers.23.mlp.experts.152.up_proj", "model.layers.23.mlp.experts.153.up_proj", "model.layers.23.mlp.experts.154.up_proj", "model.layers.23.mlp.experts.155.up_proj", "model.layers.23.mlp.experts.156.up_proj", "model.layers.23.mlp.experts.157.up_proj", "model.layers.23.mlp.experts.158.up_proj", "model.layers.23.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.9093727031722665e-06, "dbits": 5033164800 } ] }, { "idx": 140, "layers": [ "model.layers.23.mlp.experts.0.down_proj", "model.layers.23.mlp.experts.1.down_proj", "model.layers.23.mlp.experts.2.down_proj", "model.layers.23.mlp.experts.3.down_proj", "model.layers.23.mlp.experts.4.down_proj", "model.layers.23.mlp.experts.5.down_proj", "model.layers.23.mlp.experts.6.down_proj", "model.layers.23.mlp.experts.7.down_proj", "model.layers.23.mlp.experts.8.down_proj", "model.layers.23.mlp.experts.9.down_proj", "model.layers.23.mlp.experts.10.down_proj", "model.layers.23.mlp.experts.11.down_proj", "model.layers.23.mlp.experts.12.down_proj", "model.layers.23.mlp.experts.13.down_proj", "model.layers.23.mlp.experts.14.down_proj", "model.layers.23.mlp.experts.15.down_proj", "model.layers.23.mlp.experts.16.down_proj", "model.layers.23.mlp.experts.17.down_proj", "model.layers.23.mlp.experts.18.down_proj", "model.layers.23.mlp.experts.19.down_proj", "model.layers.23.mlp.experts.20.down_proj", "model.layers.23.mlp.experts.21.down_proj", "model.layers.23.mlp.experts.22.down_proj", "model.layers.23.mlp.experts.23.down_proj", "model.layers.23.mlp.experts.24.down_proj", "model.layers.23.mlp.experts.25.down_proj", "model.layers.23.mlp.experts.26.down_proj", "model.layers.23.mlp.experts.27.down_proj", "model.layers.23.mlp.experts.28.down_proj", "model.layers.23.mlp.experts.29.down_proj", "model.layers.23.mlp.experts.30.down_proj", "model.layers.23.mlp.experts.31.down_proj", "model.layers.23.mlp.experts.32.down_proj", "model.layers.23.mlp.experts.33.down_proj", "model.layers.23.mlp.experts.34.down_proj", "model.layers.23.mlp.experts.35.down_proj", "model.layers.23.mlp.experts.36.down_proj", "model.layers.23.mlp.experts.37.down_proj", "model.layers.23.mlp.experts.38.down_proj", "model.layers.23.mlp.experts.39.down_proj", "model.layers.23.mlp.experts.40.down_proj", "model.layers.23.mlp.experts.41.down_proj", "model.layers.23.mlp.experts.42.down_proj", "model.layers.23.mlp.experts.43.down_proj", "model.layers.23.mlp.experts.44.down_proj", "model.layers.23.mlp.experts.45.down_proj", "model.layers.23.mlp.experts.46.down_proj", "model.layers.23.mlp.experts.47.down_proj", "model.layers.23.mlp.experts.48.down_proj", "model.layers.23.mlp.experts.49.down_proj", "model.layers.23.mlp.experts.50.down_proj", "model.layers.23.mlp.experts.51.down_proj", "model.layers.23.mlp.experts.52.down_proj", "model.layers.23.mlp.experts.53.down_proj", "model.layers.23.mlp.experts.54.down_proj", "model.layers.23.mlp.experts.55.down_proj", "model.layers.23.mlp.experts.56.down_proj", "model.layers.23.mlp.experts.57.down_proj", "model.layers.23.mlp.experts.58.down_proj", "model.layers.23.mlp.experts.59.down_proj", "model.layers.23.mlp.experts.60.down_proj", "model.layers.23.mlp.experts.61.down_proj", "model.layers.23.mlp.experts.62.down_proj", "model.layers.23.mlp.experts.63.down_proj", "model.layers.23.mlp.experts.64.down_proj", "model.layers.23.mlp.experts.65.down_proj", "model.layers.23.mlp.experts.66.down_proj", "model.layers.23.mlp.experts.67.down_proj", "model.layers.23.mlp.experts.68.down_proj", "model.layers.23.mlp.experts.69.down_proj", "model.layers.23.mlp.experts.70.down_proj", "model.layers.23.mlp.experts.71.down_proj", "model.layers.23.mlp.experts.72.down_proj", "model.layers.23.mlp.experts.73.down_proj", "model.layers.23.mlp.experts.74.down_proj", "model.layers.23.mlp.experts.75.down_proj", "model.layers.23.mlp.experts.76.down_proj", "model.layers.23.mlp.experts.77.down_proj", "model.layers.23.mlp.experts.78.down_proj", "model.layers.23.mlp.experts.79.down_proj", "model.layers.23.mlp.experts.80.down_proj", "model.layers.23.mlp.experts.81.down_proj", "model.layers.23.mlp.experts.82.down_proj", "model.layers.23.mlp.experts.83.down_proj", "model.layers.23.mlp.experts.84.down_proj", "model.layers.23.mlp.experts.85.down_proj", "model.layers.23.mlp.experts.86.down_proj", "model.layers.23.mlp.experts.87.down_proj", "model.layers.23.mlp.experts.88.down_proj", "model.layers.23.mlp.experts.89.down_proj", "model.layers.23.mlp.experts.90.down_proj", "model.layers.23.mlp.experts.91.down_proj", "model.layers.23.mlp.experts.92.down_proj", "model.layers.23.mlp.experts.93.down_proj", "model.layers.23.mlp.experts.94.down_proj", "model.layers.23.mlp.experts.95.down_proj", "model.layers.23.mlp.experts.96.down_proj", "model.layers.23.mlp.experts.97.down_proj", "model.layers.23.mlp.experts.98.down_proj", "model.layers.23.mlp.experts.99.down_proj", "model.layers.23.mlp.experts.100.down_proj", "model.layers.23.mlp.experts.101.down_proj", "model.layers.23.mlp.experts.102.down_proj", "model.layers.23.mlp.experts.103.down_proj", "model.layers.23.mlp.experts.104.down_proj", "model.layers.23.mlp.experts.105.down_proj", "model.layers.23.mlp.experts.106.down_proj", "model.layers.23.mlp.experts.107.down_proj", "model.layers.23.mlp.experts.108.down_proj", "model.layers.23.mlp.experts.109.down_proj", "model.layers.23.mlp.experts.110.down_proj", "model.layers.23.mlp.experts.111.down_proj", "model.layers.23.mlp.experts.112.down_proj", "model.layers.23.mlp.experts.113.down_proj", "model.layers.23.mlp.experts.114.down_proj", "model.layers.23.mlp.experts.115.down_proj", "model.layers.23.mlp.experts.116.down_proj", "model.layers.23.mlp.experts.117.down_proj", "model.layers.23.mlp.experts.118.down_proj", "model.layers.23.mlp.experts.119.down_proj", "model.layers.23.mlp.experts.120.down_proj", "model.layers.23.mlp.experts.121.down_proj", "model.layers.23.mlp.experts.122.down_proj", "model.layers.23.mlp.experts.123.down_proj", "model.layers.23.mlp.experts.124.down_proj", "model.layers.23.mlp.experts.125.down_proj", "model.layers.23.mlp.experts.126.down_proj", "model.layers.23.mlp.experts.127.down_proj", "model.layers.23.mlp.experts.128.down_proj", "model.layers.23.mlp.experts.129.down_proj", "model.layers.23.mlp.experts.130.down_proj", "model.layers.23.mlp.experts.131.down_proj", "model.layers.23.mlp.experts.132.down_proj", "model.layers.23.mlp.experts.133.down_proj", "model.layers.23.mlp.experts.134.down_proj", "model.layers.23.mlp.experts.135.down_proj", "model.layers.23.mlp.experts.136.down_proj", "model.layers.23.mlp.experts.137.down_proj", "model.layers.23.mlp.experts.138.down_proj", "model.layers.23.mlp.experts.139.down_proj", "model.layers.23.mlp.experts.140.down_proj", "model.layers.23.mlp.experts.141.down_proj", "model.layers.23.mlp.experts.142.down_proj", "model.layers.23.mlp.experts.143.down_proj", "model.layers.23.mlp.experts.144.down_proj", "model.layers.23.mlp.experts.145.down_proj", "model.layers.23.mlp.experts.146.down_proj", "model.layers.23.mlp.experts.147.down_proj", "model.layers.23.mlp.experts.148.down_proj", "model.layers.23.mlp.experts.149.down_proj", "model.layers.23.mlp.experts.150.down_proj", "model.layers.23.mlp.experts.151.down_proj", "model.layers.23.mlp.experts.152.down_proj", "model.layers.23.mlp.experts.153.down_proj", "model.layers.23.mlp.experts.154.down_proj", "model.layers.23.mlp.experts.155.down_proj", "model.layers.23.mlp.experts.156.down_proj", "model.layers.23.mlp.experts.157.down_proj", "model.layers.23.mlp.experts.158.down_proj", "model.layers.23.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -6.436195690184832e-06, "dbits": 2516582400 } ] }, { "idx": 141, "layers": [ "model.layers.24.self_attn.q_proj" ], "candidates": [ { "dkld": 1.149096060544248e-05, "dbits": 125829120 } ] }, { "idx": 142, "layers": [ "model.layers.24.self_attn.k_proj", "model.layers.24.self_attn.v_proj" ], "candidates": [ { "dkld": 1.5890604117885015e-06, "dbits": 20971520 } ] }, { "idx": 143, "layers": [ "model.layers.24.self_attn.o_proj" ], "candidates": [ { "dkld": 7.212461787275883e-06, "dbits": 125829120 } ] }, { "idx": 144, "layers": [ "model.layers.24.mlp.shared_experts.gate_proj", "model.layers.24.mlp.shared_experts.up_proj", "model.layers.24.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -8.58386338222774e-06, "dbits": 47185920 } ] }, { "idx": 145, "layers": [ "model.layers.24.mlp.experts.0.gate_proj", "model.layers.24.mlp.experts.1.gate_proj", "model.layers.24.mlp.experts.2.gate_proj", "model.layers.24.mlp.experts.3.gate_proj", "model.layers.24.mlp.experts.4.gate_proj", "model.layers.24.mlp.experts.5.gate_proj", "model.layers.24.mlp.experts.6.gate_proj", "model.layers.24.mlp.experts.7.gate_proj", "model.layers.24.mlp.experts.8.gate_proj", "model.layers.24.mlp.experts.9.gate_proj", "model.layers.24.mlp.experts.10.gate_proj", "model.layers.24.mlp.experts.11.gate_proj", "model.layers.24.mlp.experts.12.gate_proj", "model.layers.24.mlp.experts.13.gate_proj", "model.layers.24.mlp.experts.14.gate_proj", "model.layers.24.mlp.experts.15.gate_proj", "model.layers.24.mlp.experts.16.gate_proj", "model.layers.24.mlp.experts.17.gate_proj", "model.layers.24.mlp.experts.18.gate_proj", "model.layers.24.mlp.experts.19.gate_proj", "model.layers.24.mlp.experts.20.gate_proj", "model.layers.24.mlp.experts.21.gate_proj", "model.layers.24.mlp.experts.22.gate_proj", "model.layers.24.mlp.experts.23.gate_proj", "model.layers.24.mlp.experts.24.gate_proj", "model.layers.24.mlp.experts.25.gate_proj", "model.layers.24.mlp.experts.26.gate_proj", "model.layers.24.mlp.experts.27.gate_proj", "model.layers.24.mlp.experts.28.gate_proj", "model.layers.24.mlp.experts.29.gate_proj", "model.layers.24.mlp.experts.30.gate_proj", "model.layers.24.mlp.experts.31.gate_proj", "model.layers.24.mlp.experts.32.gate_proj", "model.layers.24.mlp.experts.33.gate_proj", "model.layers.24.mlp.experts.34.gate_proj", "model.layers.24.mlp.experts.35.gate_proj", "model.layers.24.mlp.experts.36.gate_proj", "model.layers.24.mlp.experts.37.gate_proj", "model.layers.24.mlp.experts.38.gate_proj", "model.layers.24.mlp.experts.39.gate_proj", "model.layers.24.mlp.experts.40.gate_proj", "model.layers.24.mlp.experts.41.gate_proj", "model.layers.24.mlp.experts.42.gate_proj", "model.layers.24.mlp.experts.43.gate_proj", "model.layers.24.mlp.experts.44.gate_proj", "model.layers.24.mlp.experts.45.gate_proj", "model.layers.24.mlp.experts.46.gate_proj", "model.layers.24.mlp.experts.47.gate_proj", "model.layers.24.mlp.experts.48.gate_proj", "model.layers.24.mlp.experts.49.gate_proj", "model.layers.24.mlp.experts.50.gate_proj", "model.layers.24.mlp.experts.51.gate_proj", "model.layers.24.mlp.experts.52.gate_proj", "model.layers.24.mlp.experts.53.gate_proj", "model.layers.24.mlp.experts.54.gate_proj", "model.layers.24.mlp.experts.55.gate_proj", "model.layers.24.mlp.experts.56.gate_proj", "model.layers.24.mlp.experts.57.gate_proj", "model.layers.24.mlp.experts.58.gate_proj", "model.layers.24.mlp.experts.59.gate_proj", "model.layers.24.mlp.experts.60.gate_proj", "model.layers.24.mlp.experts.61.gate_proj", "model.layers.24.mlp.experts.62.gate_proj", "model.layers.24.mlp.experts.63.gate_proj", "model.layers.24.mlp.experts.64.gate_proj", "model.layers.24.mlp.experts.65.gate_proj", "model.layers.24.mlp.experts.66.gate_proj", "model.layers.24.mlp.experts.67.gate_proj", "model.layers.24.mlp.experts.68.gate_proj", "model.layers.24.mlp.experts.69.gate_proj", "model.layers.24.mlp.experts.70.gate_proj", "model.layers.24.mlp.experts.71.gate_proj", "model.layers.24.mlp.experts.72.gate_proj", "model.layers.24.mlp.experts.73.gate_proj", "model.layers.24.mlp.experts.74.gate_proj", "model.layers.24.mlp.experts.75.gate_proj", "model.layers.24.mlp.experts.76.gate_proj", "model.layers.24.mlp.experts.77.gate_proj", "model.layers.24.mlp.experts.78.gate_proj", "model.layers.24.mlp.experts.79.gate_proj", "model.layers.24.mlp.experts.80.gate_proj", "model.layers.24.mlp.experts.81.gate_proj", "model.layers.24.mlp.experts.82.gate_proj", "model.layers.24.mlp.experts.83.gate_proj", "model.layers.24.mlp.experts.84.gate_proj", "model.layers.24.mlp.experts.85.gate_proj", "model.layers.24.mlp.experts.86.gate_proj", "model.layers.24.mlp.experts.87.gate_proj", "model.layers.24.mlp.experts.88.gate_proj", "model.layers.24.mlp.experts.89.gate_proj", "model.layers.24.mlp.experts.90.gate_proj", "model.layers.24.mlp.experts.91.gate_proj", "model.layers.24.mlp.experts.92.gate_proj", "model.layers.24.mlp.experts.93.gate_proj", "model.layers.24.mlp.experts.94.gate_proj", "model.layers.24.mlp.experts.95.gate_proj", "model.layers.24.mlp.experts.96.gate_proj", "model.layers.24.mlp.experts.97.gate_proj", "model.layers.24.mlp.experts.98.gate_proj", "model.layers.24.mlp.experts.99.gate_proj", "model.layers.24.mlp.experts.100.gate_proj", "model.layers.24.mlp.experts.101.gate_proj", "model.layers.24.mlp.experts.102.gate_proj", "model.layers.24.mlp.experts.103.gate_proj", "model.layers.24.mlp.experts.104.gate_proj", "model.layers.24.mlp.experts.105.gate_proj", "model.layers.24.mlp.experts.106.gate_proj", "model.layers.24.mlp.experts.107.gate_proj", "model.layers.24.mlp.experts.108.gate_proj", "model.layers.24.mlp.experts.109.gate_proj", "model.layers.24.mlp.experts.110.gate_proj", "model.layers.24.mlp.experts.111.gate_proj", "model.layers.24.mlp.experts.112.gate_proj", "model.layers.24.mlp.experts.113.gate_proj", "model.layers.24.mlp.experts.114.gate_proj", "model.layers.24.mlp.experts.115.gate_proj", "model.layers.24.mlp.experts.116.gate_proj", "model.layers.24.mlp.experts.117.gate_proj", "model.layers.24.mlp.experts.118.gate_proj", "model.layers.24.mlp.experts.119.gate_proj", "model.layers.24.mlp.experts.120.gate_proj", "model.layers.24.mlp.experts.121.gate_proj", "model.layers.24.mlp.experts.122.gate_proj", "model.layers.24.mlp.experts.123.gate_proj", "model.layers.24.mlp.experts.124.gate_proj", "model.layers.24.mlp.experts.125.gate_proj", "model.layers.24.mlp.experts.126.gate_proj", "model.layers.24.mlp.experts.127.gate_proj", "model.layers.24.mlp.experts.128.gate_proj", "model.layers.24.mlp.experts.129.gate_proj", "model.layers.24.mlp.experts.130.gate_proj", "model.layers.24.mlp.experts.131.gate_proj", "model.layers.24.mlp.experts.132.gate_proj", "model.layers.24.mlp.experts.133.gate_proj", "model.layers.24.mlp.experts.134.gate_proj", "model.layers.24.mlp.experts.135.gate_proj", "model.layers.24.mlp.experts.136.gate_proj", "model.layers.24.mlp.experts.137.gate_proj", "model.layers.24.mlp.experts.138.gate_proj", "model.layers.24.mlp.experts.139.gate_proj", "model.layers.24.mlp.experts.140.gate_proj", "model.layers.24.mlp.experts.141.gate_proj", "model.layers.24.mlp.experts.142.gate_proj", "model.layers.24.mlp.experts.143.gate_proj", "model.layers.24.mlp.experts.144.gate_proj", "model.layers.24.mlp.experts.145.gate_proj", "model.layers.24.mlp.experts.146.gate_proj", "model.layers.24.mlp.experts.147.gate_proj", "model.layers.24.mlp.experts.148.gate_proj", "model.layers.24.mlp.experts.149.gate_proj", "model.layers.24.mlp.experts.150.gate_proj", "model.layers.24.mlp.experts.151.gate_proj", "model.layers.24.mlp.experts.152.gate_proj", "model.layers.24.mlp.experts.153.gate_proj", "model.layers.24.mlp.experts.154.gate_proj", "model.layers.24.mlp.experts.155.gate_proj", "model.layers.24.mlp.experts.156.gate_proj", "model.layers.24.mlp.experts.157.gate_proj", "model.layers.24.mlp.experts.158.gate_proj", "model.layers.24.mlp.experts.159.gate_proj", "model.layers.24.mlp.experts.0.up_proj", "model.layers.24.mlp.experts.1.up_proj", "model.layers.24.mlp.experts.2.up_proj", "model.layers.24.mlp.experts.3.up_proj", "model.layers.24.mlp.experts.4.up_proj", "model.layers.24.mlp.experts.5.up_proj", "model.layers.24.mlp.experts.6.up_proj", "model.layers.24.mlp.experts.7.up_proj", "model.layers.24.mlp.experts.8.up_proj", "model.layers.24.mlp.experts.9.up_proj", "model.layers.24.mlp.experts.10.up_proj", "model.layers.24.mlp.experts.11.up_proj", "model.layers.24.mlp.experts.12.up_proj", "model.layers.24.mlp.experts.13.up_proj", "model.layers.24.mlp.experts.14.up_proj", "model.layers.24.mlp.experts.15.up_proj", "model.layers.24.mlp.experts.16.up_proj", "model.layers.24.mlp.experts.17.up_proj", "model.layers.24.mlp.experts.18.up_proj", "model.layers.24.mlp.experts.19.up_proj", "model.layers.24.mlp.experts.20.up_proj", "model.layers.24.mlp.experts.21.up_proj", "model.layers.24.mlp.experts.22.up_proj", "model.layers.24.mlp.experts.23.up_proj", "model.layers.24.mlp.experts.24.up_proj", "model.layers.24.mlp.experts.25.up_proj", "model.layers.24.mlp.experts.26.up_proj", "model.layers.24.mlp.experts.27.up_proj", "model.layers.24.mlp.experts.28.up_proj", "model.layers.24.mlp.experts.29.up_proj", "model.layers.24.mlp.experts.30.up_proj", "model.layers.24.mlp.experts.31.up_proj", "model.layers.24.mlp.experts.32.up_proj", "model.layers.24.mlp.experts.33.up_proj", "model.layers.24.mlp.experts.34.up_proj", "model.layers.24.mlp.experts.35.up_proj", "model.layers.24.mlp.experts.36.up_proj", "model.layers.24.mlp.experts.37.up_proj", "model.layers.24.mlp.experts.38.up_proj", "model.layers.24.mlp.experts.39.up_proj", "model.layers.24.mlp.experts.40.up_proj", "model.layers.24.mlp.experts.41.up_proj", "model.layers.24.mlp.experts.42.up_proj", "model.layers.24.mlp.experts.43.up_proj", "model.layers.24.mlp.experts.44.up_proj", "model.layers.24.mlp.experts.45.up_proj", "model.layers.24.mlp.experts.46.up_proj", "model.layers.24.mlp.experts.47.up_proj", "model.layers.24.mlp.experts.48.up_proj", "model.layers.24.mlp.experts.49.up_proj", "model.layers.24.mlp.experts.50.up_proj", "model.layers.24.mlp.experts.51.up_proj", "model.layers.24.mlp.experts.52.up_proj", "model.layers.24.mlp.experts.53.up_proj", "model.layers.24.mlp.experts.54.up_proj", "model.layers.24.mlp.experts.55.up_proj", "model.layers.24.mlp.experts.56.up_proj", "model.layers.24.mlp.experts.57.up_proj", "model.layers.24.mlp.experts.58.up_proj", "model.layers.24.mlp.experts.59.up_proj", "model.layers.24.mlp.experts.60.up_proj", "model.layers.24.mlp.experts.61.up_proj", "model.layers.24.mlp.experts.62.up_proj", "model.layers.24.mlp.experts.63.up_proj", "model.layers.24.mlp.experts.64.up_proj", "model.layers.24.mlp.experts.65.up_proj", "model.layers.24.mlp.experts.66.up_proj", "model.layers.24.mlp.experts.67.up_proj", "model.layers.24.mlp.experts.68.up_proj", "model.layers.24.mlp.experts.69.up_proj", "model.layers.24.mlp.experts.70.up_proj", "model.layers.24.mlp.experts.71.up_proj", "model.layers.24.mlp.experts.72.up_proj", "model.layers.24.mlp.experts.73.up_proj", "model.layers.24.mlp.experts.74.up_proj", "model.layers.24.mlp.experts.75.up_proj", "model.layers.24.mlp.experts.76.up_proj", "model.layers.24.mlp.experts.77.up_proj", "model.layers.24.mlp.experts.78.up_proj", "model.layers.24.mlp.experts.79.up_proj", "model.layers.24.mlp.experts.80.up_proj", "model.layers.24.mlp.experts.81.up_proj", "model.layers.24.mlp.experts.82.up_proj", "model.layers.24.mlp.experts.83.up_proj", "model.layers.24.mlp.experts.84.up_proj", "model.layers.24.mlp.experts.85.up_proj", "model.layers.24.mlp.experts.86.up_proj", "model.layers.24.mlp.experts.87.up_proj", "model.layers.24.mlp.experts.88.up_proj", "model.layers.24.mlp.experts.89.up_proj", "model.layers.24.mlp.experts.90.up_proj", "model.layers.24.mlp.experts.91.up_proj", "model.layers.24.mlp.experts.92.up_proj", "model.layers.24.mlp.experts.93.up_proj", "model.layers.24.mlp.experts.94.up_proj", "model.layers.24.mlp.experts.95.up_proj", "model.layers.24.mlp.experts.96.up_proj", "model.layers.24.mlp.experts.97.up_proj", "model.layers.24.mlp.experts.98.up_proj", "model.layers.24.mlp.experts.99.up_proj", "model.layers.24.mlp.experts.100.up_proj", "model.layers.24.mlp.experts.101.up_proj", "model.layers.24.mlp.experts.102.up_proj", "model.layers.24.mlp.experts.103.up_proj", "model.layers.24.mlp.experts.104.up_proj", "model.layers.24.mlp.experts.105.up_proj", "model.layers.24.mlp.experts.106.up_proj", "model.layers.24.mlp.experts.107.up_proj", "model.layers.24.mlp.experts.108.up_proj", "model.layers.24.mlp.experts.109.up_proj", "model.layers.24.mlp.experts.110.up_proj", "model.layers.24.mlp.experts.111.up_proj", "model.layers.24.mlp.experts.112.up_proj", "model.layers.24.mlp.experts.113.up_proj", "model.layers.24.mlp.experts.114.up_proj", "model.layers.24.mlp.experts.115.up_proj", "model.layers.24.mlp.experts.116.up_proj", "model.layers.24.mlp.experts.117.up_proj", "model.layers.24.mlp.experts.118.up_proj", "model.layers.24.mlp.experts.119.up_proj", "model.layers.24.mlp.experts.120.up_proj", "model.layers.24.mlp.experts.121.up_proj", "model.layers.24.mlp.experts.122.up_proj", "model.layers.24.mlp.experts.123.up_proj", "model.layers.24.mlp.experts.124.up_proj", "model.layers.24.mlp.experts.125.up_proj", "model.layers.24.mlp.experts.126.up_proj", "model.layers.24.mlp.experts.127.up_proj", "model.layers.24.mlp.experts.128.up_proj", "model.layers.24.mlp.experts.129.up_proj", "model.layers.24.mlp.experts.130.up_proj", "model.layers.24.mlp.experts.131.up_proj", "model.layers.24.mlp.experts.132.up_proj", "model.layers.24.mlp.experts.133.up_proj", "model.layers.24.mlp.experts.134.up_proj", "model.layers.24.mlp.experts.135.up_proj", "model.layers.24.mlp.experts.136.up_proj", "model.layers.24.mlp.experts.137.up_proj", "model.layers.24.mlp.experts.138.up_proj", "model.layers.24.mlp.experts.139.up_proj", "model.layers.24.mlp.experts.140.up_proj", "model.layers.24.mlp.experts.141.up_proj", "model.layers.24.mlp.experts.142.up_proj", "model.layers.24.mlp.experts.143.up_proj", "model.layers.24.mlp.experts.144.up_proj", "model.layers.24.mlp.experts.145.up_proj", "model.layers.24.mlp.experts.146.up_proj", "model.layers.24.mlp.experts.147.up_proj", "model.layers.24.mlp.experts.148.up_proj", "model.layers.24.mlp.experts.149.up_proj", "model.layers.24.mlp.experts.150.up_proj", "model.layers.24.mlp.experts.151.up_proj", "model.layers.24.mlp.experts.152.up_proj", "model.layers.24.mlp.experts.153.up_proj", "model.layers.24.mlp.experts.154.up_proj", "model.layers.24.mlp.experts.155.up_proj", "model.layers.24.mlp.experts.156.up_proj", "model.layers.24.mlp.experts.157.up_proj", "model.layers.24.mlp.experts.158.up_proj", "model.layers.24.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 6.112793926149173e-07, "dbits": 5033164800 } ] }, { "idx": 146, "layers": [ "model.layers.24.mlp.experts.0.down_proj", "model.layers.24.mlp.experts.1.down_proj", "model.layers.24.mlp.experts.2.down_proj", "model.layers.24.mlp.experts.3.down_proj", "model.layers.24.mlp.experts.4.down_proj", "model.layers.24.mlp.experts.5.down_proj", "model.layers.24.mlp.experts.6.down_proj", "model.layers.24.mlp.experts.7.down_proj", "model.layers.24.mlp.experts.8.down_proj", "model.layers.24.mlp.experts.9.down_proj", "model.layers.24.mlp.experts.10.down_proj", "model.layers.24.mlp.experts.11.down_proj", "model.layers.24.mlp.experts.12.down_proj", "model.layers.24.mlp.experts.13.down_proj", "model.layers.24.mlp.experts.14.down_proj", "model.layers.24.mlp.experts.15.down_proj", "model.layers.24.mlp.experts.16.down_proj", "model.layers.24.mlp.experts.17.down_proj", "model.layers.24.mlp.experts.18.down_proj", "model.layers.24.mlp.experts.19.down_proj", "model.layers.24.mlp.experts.20.down_proj", "model.layers.24.mlp.experts.21.down_proj", "model.layers.24.mlp.experts.22.down_proj", "model.layers.24.mlp.experts.23.down_proj", "model.layers.24.mlp.experts.24.down_proj", "model.layers.24.mlp.experts.25.down_proj", "model.layers.24.mlp.experts.26.down_proj", "model.layers.24.mlp.experts.27.down_proj", "model.layers.24.mlp.experts.28.down_proj", "model.layers.24.mlp.experts.29.down_proj", "model.layers.24.mlp.experts.30.down_proj", "model.layers.24.mlp.experts.31.down_proj", "model.layers.24.mlp.experts.32.down_proj", "model.layers.24.mlp.experts.33.down_proj", "model.layers.24.mlp.experts.34.down_proj", "model.layers.24.mlp.experts.35.down_proj", "model.layers.24.mlp.experts.36.down_proj", "model.layers.24.mlp.experts.37.down_proj", "model.layers.24.mlp.experts.38.down_proj", "model.layers.24.mlp.experts.39.down_proj", "model.layers.24.mlp.experts.40.down_proj", "model.layers.24.mlp.experts.41.down_proj", "model.layers.24.mlp.experts.42.down_proj", "model.layers.24.mlp.experts.43.down_proj", "model.layers.24.mlp.experts.44.down_proj", "model.layers.24.mlp.experts.45.down_proj", "model.layers.24.mlp.experts.46.down_proj", "model.layers.24.mlp.experts.47.down_proj", "model.layers.24.mlp.experts.48.down_proj", "model.layers.24.mlp.experts.49.down_proj", "model.layers.24.mlp.experts.50.down_proj", "model.layers.24.mlp.experts.51.down_proj", "model.layers.24.mlp.experts.52.down_proj", "model.layers.24.mlp.experts.53.down_proj", "model.layers.24.mlp.experts.54.down_proj", "model.layers.24.mlp.experts.55.down_proj", "model.layers.24.mlp.experts.56.down_proj", "model.layers.24.mlp.experts.57.down_proj", "model.layers.24.mlp.experts.58.down_proj", "model.layers.24.mlp.experts.59.down_proj", "model.layers.24.mlp.experts.60.down_proj", "model.layers.24.mlp.experts.61.down_proj", "model.layers.24.mlp.experts.62.down_proj", "model.layers.24.mlp.experts.63.down_proj", "model.layers.24.mlp.experts.64.down_proj", "model.layers.24.mlp.experts.65.down_proj", "model.layers.24.mlp.experts.66.down_proj", "model.layers.24.mlp.experts.67.down_proj", "model.layers.24.mlp.experts.68.down_proj", "model.layers.24.mlp.experts.69.down_proj", "model.layers.24.mlp.experts.70.down_proj", "model.layers.24.mlp.experts.71.down_proj", "model.layers.24.mlp.experts.72.down_proj", "model.layers.24.mlp.experts.73.down_proj", "model.layers.24.mlp.experts.74.down_proj", "model.layers.24.mlp.experts.75.down_proj", "model.layers.24.mlp.experts.76.down_proj", "model.layers.24.mlp.experts.77.down_proj", "model.layers.24.mlp.experts.78.down_proj", "model.layers.24.mlp.experts.79.down_proj", "model.layers.24.mlp.experts.80.down_proj", "model.layers.24.mlp.experts.81.down_proj", "model.layers.24.mlp.experts.82.down_proj", "model.layers.24.mlp.experts.83.down_proj", "model.layers.24.mlp.experts.84.down_proj", "model.layers.24.mlp.experts.85.down_proj", "model.layers.24.mlp.experts.86.down_proj", "model.layers.24.mlp.experts.87.down_proj", "model.layers.24.mlp.experts.88.down_proj", "model.layers.24.mlp.experts.89.down_proj", "model.layers.24.mlp.experts.90.down_proj", "model.layers.24.mlp.experts.91.down_proj", "model.layers.24.mlp.experts.92.down_proj", "model.layers.24.mlp.experts.93.down_proj", "model.layers.24.mlp.experts.94.down_proj", "model.layers.24.mlp.experts.95.down_proj", "model.layers.24.mlp.experts.96.down_proj", "model.layers.24.mlp.experts.97.down_proj", "model.layers.24.mlp.experts.98.down_proj", "model.layers.24.mlp.experts.99.down_proj", "model.layers.24.mlp.experts.100.down_proj", "model.layers.24.mlp.experts.101.down_proj", "model.layers.24.mlp.experts.102.down_proj", "model.layers.24.mlp.experts.103.down_proj", "model.layers.24.mlp.experts.104.down_proj", "model.layers.24.mlp.experts.105.down_proj", "model.layers.24.mlp.experts.106.down_proj", "model.layers.24.mlp.experts.107.down_proj", "model.layers.24.mlp.experts.108.down_proj", "model.layers.24.mlp.experts.109.down_proj", "model.layers.24.mlp.experts.110.down_proj", "model.layers.24.mlp.experts.111.down_proj", "model.layers.24.mlp.experts.112.down_proj", "model.layers.24.mlp.experts.113.down_proj", "model.layers.24.mlp.experts.114.down_proj", "model.layers.24.mlp.experts.115.down_proj", "model.layers.24.mlp.experts.116.down_proj", "model.layers.24.mlp.experts.117.down_proj", "model.layers.24.mlp.experts.118.down_proj", "model.layers.24.mlp.experts.119.down_proj", "model.layers.24.mlp.experts.120.down_proj", "model.layers.24.mlp.experts.121.down_proj", "model.layers.24.mlp.experts.122.down_proj", "model.layers.24.mlp.experts.123.down_proj", "model.layers.24.mlp.experts.124.down_proj", "model.layers.24.mlp.experts.125.down_proj", "model.layers.24.mlp.experts.126.down_proj", "model.layers.24.mlp.experts.127.down_proj", "model.layers.24.mlp.experts.128.down_proj", "model.layers.24.mlp.experts.129.down_proj", "model.layers.24.mlp.experts.130.down_proj", "model.layers.24.mlp.experts.131.down_proj", "model.layers.24.mlp.experts.132.down_proj", "model.layers.24.mlp.experts.133.down_proj", "model.layers.24.mlp.experts.134.down_proj", "model.layers.24.mlp.experts.135.down_proj", "model.layers.24.mlp.experts.136.down_proj", "model.layers.24.mlp.experts.137.down_proj", "model.layers.24.mlp.experts.138.down_proj", "model.layers.24.mlp.experts.139.down_proj", "model.layers.24.mlp.experts.140.down_proj", "model.layers.24.mlp.experts.141.down_proj", "model.layers.24.mlp.experts.142.down_proj", "model.layers.24.mlp.experts.143.down_proj", "model.layers.24.mlp.experts.144.down_proj", "model.layers.24.mlp.experts.145.down_proj", "model.layers.24.mlp.experts.146.down_proj", "model.layers.24.mlp.experts.147.down_proj", "model.layers.24.mlp.experts.148.down_proj", "model.layers.24.mlp.experts.149.down_proj", "model.layers.24.mlp.experts.150.down_proj", "model.layers.24.mlp.experts.151.down_proj", "model.layers.24.mlp.experts.152.down_proj", "model.layers.24.mlp.experts.153.down_proj", "model.layers.24.mlp.experts.154.down_proj", "model.layers.24.mlp.experts.155.down_proj", "model.layers.24.mlp.experts.156.down_proj", "model.layers.24.mlp.experts.157.down_proj", "model.layers.24.mlp.experts.158.down_proj", "model.layers.24.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -2.461276017129443e-06, "dbits": 2516582400 } ] }, { "idx": 147, "layers": [ "model.layers.25.self_attn.q_proj" ], "candidates": [ { "dkld": -9.771907934918978e-06, "dbits": 125829120 } ] }, { "idx": 148, "layers": [ "model.layers.25.self_attn.k_proj", "model.layers.25.self_attn.v_proj" ], "candidates": [ { "dkld": 1.0639068204909606e-05, "dbits": 20971520 } ] }, { "idx": 149, "layers": [ "model.layers.25.self_attn.o_proj" ], "candidates": [ { "dkld": 7.675116648897529e-07, "dbits": 125829120 } ] }, { "idx": 150, "layers": [ "model.layers.25.mlp.shared_experts.gate_proj", "model.layers.25.mlp.shared_experts.up_proj", "model.layers.25.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -4.708999767899513e-07, "dbits": 47185920 } ] }, { "idx": 151, "layers": [ "model.layers.25.mlp.experts.0.gate_proj", "model.layers.25.mlp.experts.1.gate_proj", "model.layers.25.mlp.experts.2.gate_proj", "model.layers.25.mlp.experts.3.gate_proj", "model.layers.25.mlp.experts.4.gate_proj", "model.layers.25.mlp.experts.5.gate_proj", "model.layers.25.mlp.experts.6.gate_proj", "model.layers.25.mlp.experts.7.gate_proj", "model.layers.25.mlp.experts.8.gate_proj", "model.layers.25.mlp.experts.9.gate_proj", "model.layers.25.mlp.experts.10.gate_proj", "model.layers.25.mlp.experts.11.gate_proj", "model.layers.25.mlp.experts.12.gate_proj", "model.layers.25.mlp.experts.13.gate_proj", "model.layers.25.mlp.experts.14.gate_proj", "model.layers.25.mlp.experts.15.gate_proj", "model.layers.25.mlp.experts.16.gate_proj", "model.layers.25.mlp.experts.17.gate_proj", "model.layers.25.mlp.experts.18.gate_proj", "model.layers.25.mlp.experts.19.gate_proj", "model.layers.25.mlp.experts.20.gate_proj", "model.layers.25.mlp.experts.21.gate_proj", "model.layers.25.mlp.experts.22.gate_proj", "model.layers.25.mlp.experts.23.gate_proj", "model.layers.25.mlp.experts.24.gate_proj", "model.layers.25.mlp.experts.25.gate_proj", "model.layers.25.mlp.experts.26.gate_proj", "model.layers.25.mlp.experts.27.gate_proj", "model.layers.25.mlp.experts.28.gate_proj", "model.layers.25.mlp.experts.29.gate_proj", "model.layers.25.mlp.experts.30.gate_proj", "model.layers.25.mlp.experts.31.gate_proj", "model.layers.25.mlp.experts.32.gate_proj", "model.layers.25.mlp.experts.33.gate_proj", "model.layers.25.mlp.experts.34.gate_proj", "model.layers.25.mlp.experts.35.gate_proj", "model.layers.25.mlp.experts.36.gate_proj", "model.layers.25.mlp.experts.37.gate_proj", "model.layers.25.mlp.experts.38.gate_proj", "model.layers.25.mlp.experts.39.gate_proj", "model.layers.25.mlp.experts.40.gate_proj", "model.layers.25.mlp.experts.41.gate_proj", "model.layers.25.mlp.experts.42.gate_proj", "model.layers.25.mlp.experts.43.gate_proj", "model.layers.25.mlp.experts.44.gate_proj", "model.layers.25.mlp.experts.45.gate_proj", "model.layers.25.mlp.experts.46.gate_proj", "model.layers.25.mlp.experts.47.gate_proj", "model.layers.25.mlp.experts.48.gate_proj", "model.layers.25.mlp.experts.49.gate_proj", "model.layers.25.mlp.experts.50.gate_proj", "model.layers.25.mlp.experts.51.gate_proj", "model.layers.25.mlp.experts.52.gate_proj", "model.layers.25.mlp.experts.53.gate_proj", "model.layers.25.mlp.experts.54.gate_proj", "model.layers.25.mlp.experts.55.gate_proj", "model.layers.25.mlp.experts.56.gate_proj", "model.layers.25.mlp.experts.57.gate_proj", "model.layers.25.mlp.experts.58.gate_proj", "model.layers.25.mlp.experts.59.gate_proj", "model.layers.25.mlp.experts.60.gate_proj", "model.layers.25.mlp.experts.61.gate_proj", "model.layers.25.mlp.experts.62.gate_proj", "model.layers.25.mlp.experts.63.gate_proj", "model.layers.25.mlp.experts.64.gate_proj", "model.layers.25.mlp.experts.65.gate_proj", "model.layers.25.mlp.experts.66.gate_proj", "model.layers.25.mlp.experts.67.gate_proj", "model.layers.25.mlp.experts.68.gate_proj", "model.layers.25.mlp.experts.69.gate_proj", "model.layers.25.mlp.experts.70.gate_proj", "model.layers.25.mlp.experts.71.gate_proj", "model.layers.25.mlp.experts.72.gate_proj", "model.layers.25.mlp.experts.73.gate_proj", "model.layers.25.mlp.experts.74.gate_proj", "model.layers.25.mlp.experts.75.gate_proj", "model.layers.25.mlp.experts.76.gate_proj", "model.layers.25.mlp.experts.77.gate_proj", "model.layers.25.mlp.experts.78.gate_proj", "model.layers.25.mlp.experts.79.gate_proj", "model.layers.25.mlp.experts.80.gate_proj", "model.layers.25.mlp.experts.81.gate_proj", "model.layers.25.mlp.experts.82.gate_proj", "model.layers.25.mlp.experts.83.gate_proj", "model.layers.25.mlp.experts.84.gate_proj", "model.layers.25.mlp.experts.85.gate_proj", "model.layers.25.mlp.experts.86.gate_proj", "model.layers.25.mlp.experts.87.gate_proj", "model.layers.25.mlp.experts.88.gate_proj", "model.layers.25.mlp.experts.89.gate_proj", "model.layers.25.mlp.experts.90.gate_proj", "model.layers.25.mlp.experts.91.gate_proj", "model.layers.25.mlp.experts.92.gate_proj", "model.layers.25.mlp.experts.93.gate_proj", "model.layers.25.mlp.experts.94.gate_proj", "model.layers.25.mlp.experts.95.gate_proj", "model.layers.25.mlp.experts.96.gate_proj", "model.layers.25.mlp.experts.97.gate_proj", "model.layers.25.mlp.experts.98.gate_proj", "model.layers.25.mlp.experts.99.gate_proj", "model.layers.25.mlp.experts.100.gate_proj", "model.layers.25.mlp.experts.101.gate_proj", "model.layers.25.mlp.experts.102.gate_proj", "model.layers.25.mlp.experts.103.gate_proj", "model.layers.25.mlp.experts.104.gate_proj", "model.layers.25.mlp.experts.105.gate_proj", "model.layers.25.mlp.experts.106.gate_proj", "model.layers.25.mlp.experts.107.gate_proj", "model.layers.25.mlp.experts.108.gate_proj", "model.layers.25.mlp.experts.109.gate_proj", "model.layers.25.mlp.experts.110.gate_proj", "model.layers.25.mlp.experts.111.gate_proj", "model.layers.25.mlp.experts.112.gate_proj", "model.layers.25.mlp.experts.113.gate_proj", "model.layers.25.mlp.experts.114.gate_proj", "model.layers.25.mlp.experts.115.gate_proj", "model.layers.25.mlp.experts.116.gate_proj", "model.layers.25.mlp.experts.117.gate_proj", "model.layers.25.mlp.experts.118.gate_proj", "model.layers.25.mlp.experts.119.gate_proj", "model.layers.25.mlp.experts.120.gate_proj", "model.layers.25.mlp.experts.121.gate_proj", "model.layers.25.mlp.experts.122.gate_proj", "model.layers.25.mlp.experts.123.gate_proj", "model.layers.25.mlp.experts.124.gate_proj", "model.layers.25.mlp.experts.125.gate_proj", "model.layers.25.mlp.experts.126.gate_proj", "model.layers.25.mlp.experts.127.gate_proj", "model.layers.25.mlp.experts.128.gate_proj", "model.layers.25.mlp.experts.129.gate_proj", "model.layers.25.mlp.experts.130.gate_proj", "model.layers.25.mlp.experts.131.gate_proj", "model.layers.25.mlp.experts.132.gate_proj", "model.layers.25.mlp.experts.133.gate_proj", "model.layers.25.mlp.experts.134.gate_proj", "model.layers.25.mlp.experts.135.gate_proj", "model.layers.25.mlp.experts.136.gate_proj", "model.layers.25.mlp.experts.137.gate_proj", "model.layers.25.mlp.experts.138.gate_proj", "model.layers.25.mlp.experts.139.gate_proj", "model.layers.25.mlp.experts.140.gate_proj", "model.layers.25.mlp.experts.141.gate_proj", "model.layers.25.mlp.experts.142.gate_proj", "model.layers.25.mlp.experts.143.gate_proj", "model.layers.25.mlp.experts.144.gate_proj", "model.layers.25.mlp.experts.145.gate_proj", "model.layers.25.mlp.experts.146.gate_proj", "model.layers.25.mlp.experts.147.gate_proj", "model.layers.25.mlp.experts.148.gate_proj", "model.layers.25.mlp.experts.149.gate_proj", "model.layers.25.mlp.experts.150.gate_proj", "model.layers.25.mlp.experts.151.gate_proj", "model.layers.25.mlp.experts.152.gate_proj", "model.layers.25.mlp.experts.153.gate_proj", "model.layers.25.mlp.experts.154.gate_proj", "model.layers.25.mlp.experts.155.gate_proj", "model.layers.25.mlp.experts.156.gate_proj", "model.layers.25.mlp.experts.157.gate_proj", "model.layers.25.mlp.experts.158.gate_proj", "model.layers.25.mlp.experts.159.gate_proj", "model.layers.25.mlp.experts.0.up_proj", "model.layers.25.mlp.experts.1.up_proj", "model.layers.25.mlp.experts.2.up_proj", "model.layers.25.mlp.experts.3.up_proj", "model.layers.25.mlp.experts.4.up_proj", "model.layers.25.mlp.experts.5.up_proj", "model.layers.25.mlp.experts.6.up_proj", "model.layers.25.mlp.experts.7.up_proj", "model.layers.25.mlp.experts.8.up_proj", "model.layers.25.mlp.experts.9.up_proj", "model.layers.25.mlp.experts.10.up_proj", "model.layers.25.mlp.experts.11.up_proj", "model.layers.25.mlp.experts.12.up_proj", "model.layers.25.mlp.experts.13.up_proj", "model.layers.25.mlp.experts.14.up_proj", "model.layers.25.mlp.experts.15.up_proj", "model.layers.25.mlp.experts.16.up_proj", "model.layers.25.mlp.experts.17.up_proj", "model.layers.25.mlp.experts.18.up_proj", "model.layers.25.mlp.experts.19.up_proj", "model.layers.25.mlp.experts.20.up_proj", "model.layers.25.mlp.experts.21.up_proj", "model.layers.25.mlp.experts.22.up_proj", "model.layers.25.mlp.experts.23.up_proj", "model.layers.25.mlp.experts.24.up_proj", "model.layers.25.mlp.experts.25.up_proj", "model.layers.25.mlp.experts.26.up_proj", "model.layers.25.mlp.experts.27.up_proj", "model.layers.25.mlp.experts.28.up_proj", "model.layers.25.mlp.experts.29.up_proj", "model.layers.25.mlp.experts.30.up_proj", "model.layers.25.mlp.experts.31.up_proj", "model.layers.25.mlp.experts.32.up_proj", "model.layers.25.mlp.experts.33.up_proj", "model.layers.25.mlp.experts.34.up_proj", "model.layers.25.mlp.experts.35.up_proj", "model.layers.25.mlp.experts.36.up_proj", "model.layers.25.mlp.experts.37.up_proj", "model.layers.25.mlp.experts.38.up_proj", "model.layers.25.mlp.experts.39.up_proj", "model.layers.25.mlp.experts.40.up_proj", "model.layers.25.mlp.experts.41.up_proj", "model.layers.25.mlp.experts.42.up_proj", "model.layers.25.mlp.experts.43.up_proj", "model.layers.25.mlp.experts.44.up_proj", "model.layers.25.mlp.experts.45.up_proj", "model.layers.25.mlp.experts.46.up_proj", "model.layers.25.mlp.experts.47.up_proj", "model.layers.25.mlp.experts.48.up_proj", "model.layers.25.mlp.experts.49.up_proj", "model.layers.25.mlp.experts.50.up_proj", "model.layers.25.mlp.experts.51.up_proj", "model.layers.25.mlp.experts.52.up_proj", "model.layers.25.mlp.experts.53.up_proj", "model.layers.25.mlp.experts.54.up_proj", "model.layers.25.mlp.experts.55.up_proj", "model.layers.25.mlp.experts.56.up_proj", "model.layers.25.mlp.experts.57.up_proj", "model.layers.25.mlp.experts.58.up_proj", "model.layers.25.mlp.experts.59.up_proj", "model.layers.25.mlp.experts.60.up_proj", "model.layers.25.mlp.experts.61.up_proj", "model.layers.25.mlp.experts.62.up_proj", "model.layers.25.mlp.experts.63.up_proj", "model.layers.25.mlp.experts.64.up_proj", "model.layers.25.mlp.experts.65.up_proj", "model.layers.25.mlp.experts.66.up_proj", "model.layers.25.mlp.experts.67.up_proj", "model.layers.25.mlp.experts.68.up_proj", "model.layers.25.mlp.experts.69.up_proj", "model.layers.25.mlp.experts.70.up_proj", "model.layers.25.mlp.experts.71.up_proj", "model.layers.25.mlp.experts.72.up_proj", "model.layers.25.mlp.experts.73.up_proj", "model.layers.25.mlp.experts.74.up_proj", "model.layers.25.mlp.experts.75.up_proj", "model.layers.25.mlp.experts.76.up_proj", "model.layers.25.mlp.experts.77.up_proj", "model.layers.25.mlp.experts.78.up_proj", "model.layers.25.mlp.experts.79.up_proj", "model.layers.25.mlp.experts.80.up_proj", "model.layers.25.mlp.experts.81.up_proj", "model.layers.25.mlp.experts.82.up_proj", "model.layers.25.mlp.experts.83.up_proj", "model.layers.25.mlp.experts.84.up_proj", "model.layers.25.mlp.experts.85.up_proj", "model.layers.25.mlp.experts.86.up_proj", "model.layers.25.mlp.experts.87.up_proj", "model.layers.25.mlp.experts.88.up_proj", "model.layers.25.mlp.experts.89.up_proj", "model.layers.25.mlp.experts.90.up_proj", "model.layers.25.mlp.experts.91.up_proj", "model.layers.25.mlp.experts.92.up_proj", "model.layers.25.mlp.experts.93.up_proj", "model.layers.25.mlp.experts.94.up_proj", "model.layers.25.mlp.experts.95.up_proj", "model.layers.25.mlp.experts.96.up_proj", "model.layers.25.mlp.experts.97.up_proj", "model.layers.25.mlp.experts.98.up_proj", "model.layers.25.mlp.experts.99.up_proj", "model.layers.25.mlp.experts.100.up_proj", "model.layers.25.mlp.experts.101.up_proj", "model.layers.25.mlp.experts.102.up_proj", "model.layers.25.mlp.experts.103.up_proj", "model.layers.25.mlp.experts.104.up_proj", "model.layers.25.mlp.experts.105.up_proj", "model.layers.25.mlp.experts.106.up_proj", "model.layers.25.mlp.experts.107.up_proj", "model.layers.25.mlp.experts.108.up_proj", "model.layers.25.mlp.experts.109.up_proj", "model.layers.25.mlp.experts.110.up_proj", "model.layers.25.mlp.experts.111.up_proj", "model.layers.25.mlp.experts.112.up_proj", "model.layers.25.mlp.experts.113.up_proj", "model.layers.25.mlp.experts.114.up_proj", "model.layers.25.mlp.experts.115.up_proj", "model.layers.25.mlp.experts.116.up_proj", "model.layers.25.mlp.experts.117.up_proj", "model.layers.25.mlp.experts.118.up_proj", "model.layers.25.mlp.experts.119.up_proj", "model.layers.25.mlp.experts.120.up_proj", "model.layers.25.mlp.experts.121.up_proj", "model.layers.25.mlp.experts.122.up_proj", "model.layers.25.mlp.experts.123.up_proj", "model.layers.25.mlp.experts.124.up_proj", "model.layers.25.mlp.experts.125.up_proj", "model.layers.25.mlp.experts.126.up_proj", "model.layers.25.mlp.experts.127.up_proj", "model.layers.25.mlp.experts.128.up_proj", "model.layers.25.mlp.experts.129.up_proj", "model.layers.25.mlp.experts.130.up_proj", "model.layers.25.mlp.experts.131.up_proj", "model.layers.25.mlp.experts.132.up_proj", "model.layers.25.mlp.experts.133.up_proj", "model.layers.25.mlp.experts.134.up_proj", "model.layers.25.mlp.experts.135.up_proj", "model.layers.25.mlp.experts.136.up_proj", "model.layers.25.mlp.experts.137.up_proj", "model.layers.25.mlp.experts.138.up_proj", "model.layers.25.mlp.experts.139.up_proj", "model.layers.25.mlp.experts.140.up_proj", "model.layers.25.mlp.experts.141.up_proj", "model.layers.25.mlp.experts.142.up_proj", "model.layers.25.mlp.experts.143.up_proj", "model.layers.25.mlp.experts.144.up_proj", "model.layers.25.mlp.experts.145.up_proj", "model.layers.25.mlp.experts.146.up_proj", "model.layers.25.mlp.experts.147.up_proj", "model.layers.25.mlp.experts.148.up_proj", "model.layers.25.mlp.experts.149.up_proj", "model.layers.25.mlp.experts.150.up_proj", "model.layers.25.mlp.experts.151.up_proj", "model.layers.25.mlp.experts.152.up_proj", "model.layers.25.mlp.experts.153.up_proj", "model.layers.25.mlp.experts.154.up_proj", "model.layers.25.mlp.experts.155.up_proj", "model.layers.25.mlp.experts.156.up_proj", "model.layers.25.mlp.experts.157.up_proj", "model.layers.25.mlp.experts.158.up_proj", "model.layers.25.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.8338464694098505e-06, "dbits": 5033164800 } ] }, { "idx": 152, "layers": [ "model.layers.25.mlp.experts.0.down_proj", "model.layers.25.mlp.experts.1.down_proj", "model.layers.25.mlp.experts.2.down_proj", "model.layers.25.mlp.experts.3.down_proj", "model.layers.25.mlp.experts.4.down_proj", "model.layers.25.mlp.experts.5.down_proj", "model.layers.25.mlp.experts.6.down_proj", "model.layers.25.mlp.experts.7.down_proj", "model.layers.25.mlp.experts.8.down_proj", "model.layers.25.mlp.experts.9.down_proj", "model.layers.25.mlp.experts.10.down_proj", "model.layers.25.mlp.experts.11.down_proj", "model.layers.25.mlp.experts.12.down_proj", "model.layers.25.mlp.experts.13.down_proj", "model.layers.25.mlp.experts.14.down_proj", "model.layers.25.mlp.experts.15.down_proj", "model.layers.25.mlp.experts.16.down_proj", "model.layers.25.mlp.experts.17.down_proj", "model.layers.25.mlp.experts.18.down_proj", "model.layers.25.mlp.experts.19.down_proj", "model.layers.25.mlp.experts.20.down_proj", "model.layers.25.mlp.experts.21.down_proj", "model.layers.25.mlp.experts.22.down_proj", "model.layers.25.mlp.experts.23.down_proj", "model.layers.25.mlp.experts.24.down_proj", "model.layers.25.mlp.experts.25.down_proj", "model.layers.25.mlp.experts.26.down_proj", "model.layers.25.mlp.experts.27.down_proj", "model.layers.25.mlp.experts.28.down_proj", "model.layers.25.mlp.experts.29.down_proj", "model.layers.25.mlp.experts.30.down_proj", "model.layers.25.mlp.experts.31.down_proj", "model.layers.25.mlp.experts.32.down_proj", "model.layers.25.mlp.experts.33.down_proj", "model.layers.25.mlp.experts.34.down_proj", "model.layers.25.mlp.experts.35.down_proj", "model.layers.25.mlp.experts.36.down_proj", "model.layers.25.mlp.experts.37.down_proj", "model.layers.25.mlp.experts.38.down_proj", "model.layers.25.mlp.experts.39.down_proj", "model.layers.25.mlp.experts.40.down_proj", "model.layers.25.mlp.experts.41.down_proj", "model.layers.25.mlp.experts.42.down_proj", "model.layers.25.mlp.experts.43.down_proj", "model.layers.25.mlp.experts.44.down_proj", "model.layers.25.mlp.experts.45.down_proj", "model.layers.25.mlp.experts.46.down_proj", "model.layers.25.mlp.experts.47.down_proj", "model.layers.25.mlp.experts.48.down_proj", "model.layers.25.mlp.experts.49.down_proj", "model.layers.25.mlp.experts.50.down_proj", "model.layers.25.mlp.experts.51.down_proj", "model.layers.25.mlp.experts.52.down_proj", "model.layers.25.mlp.experts.53.down_proj", "model.layers.25.mlp.experts.54.down_proj", "model.layers.25.mlp.experts.55.down_proj", "model.layers.25.mlp.experts.56.down_proj", "model.layers.25.mlp.experts.57.down_proj", "model.layers.25.mlp.experts.58.down_proj", "model.layers.25.mlp.experts.59.down_proj", "model.layers.25.mlp.experts.60.down_proj", "model.layers.25.mlp.experts.61.down_proj", "model.layers.25.mlp.experts.62.down_proj", "model.layers.25.mlp.experts.63.down_proj", "model.layers.25.mlp.experts.64.down_proj", "model.layers.25.mlp.experts.65.down_proj", "model.layers.25.mlp.experts.66.down_proj", "model.layers.25.mlp.experts.67.down_proj", "model.layers.25.mlp.experts.68.down_proj", "model.layers.25.mlp.experts.69.down_proj", "model.layers.25.mlp.experts.70.down_proj", "model.layers.25.mlp.experts.71.down_proj", "model.layers.25.mlp.experts.72.down_proj", "model.layers.25.mlp.experts.73.down_proj", "model.layers.25.mlp.experts.74.down_proj", "model.layers.25.mlp.experts.75.down_proj", "model.layers.25.mlp.experts.76.down_proj", "model.layers.25.mlp.experts.77.down_proj", "model.layers.25.mlp.experts.78.down_proj", "model.layers.25.mlp.experts.79.down_proj", "model.layers.25.mlp.experts.80.down_proj", "model.layers.25.mlp.experts.81.down_proj", "model.layers.25.mlp.experts.82.down_proj", "model.layers.25.mlp.experts.83.down_proj", "model.layers.25.mlp.experts.84.down_proj", "model.layers.25.mlp.experts.85.down_proj", "model.layers.25.mlp.experts.86.down_proj", "model.layers.25.mlp.experts.87.down_proj", "model.layers.25.mlp.experts.88.down_proj", "model.layers.25.mlp.experts.89.down_proj", "model.layers.25.mlp.experts.90.down_proj", "model.layers.25.mlp.experts.91.down_proj", "model.layers.25.mlp.experts.92.down_proj", "model.layers.25.mlp.experts.93.down_proj", "model.layers.25.mlp.experts.94.down_proj", "model.layers.25.mlp.experts.95.down_proj", "model.layers.25.mlp.experts.96.down_proj", "model.layers.25.mlp.experts.97.down_proj", "model.layers.25.mlp.experts.98.down_proj", "model.layers.25.mlp.experts.99.down_proj", "model.layers.25.mlp.experts.100.down_proj", "model.layers.25.mlp.experts.101.down_proj", "model.layers.25.mlp.experts.102.down_proj", "model.layers.25.mlp.experts.103.down_proj", "model.layers.25.mlp.experts.104.down_proj", "model.layers.25.mlp.experts.105.down_proj", "model.layers.25.mlp.experts.106.down_proj", "model.layers.25.mlp.experts.107.down_proj", "model.layers.25.mlp.experts.108.down_proj", "model.layers.25.mlp.experts.109.down_proj", "model.layers.25.mlp.experts.110.down_proj", "model.layers.25.mlp.experts.111.down_proj", "model.layers.25.mlp.experts.112.down_proj", "model.layers.25.mlp.experts.113.down_proj", "model.layers.25.mlp.experts.114.down_proj", "model.layers.25.mlp.experts.115.down_proj", "model.layers.25.mlp.experts.116.down_proj", "model.layers.25.mlp.experts.117.down_proj", "model.layers.25.mlp.experts.118.down_proj", "model.layers.25.mlp.experts.119.down_proj", "model.layers.25.mlp.experts.120.down_proj", "model.layers.25.mlp.experts.121.down_proj", "model.layers.25.mlp.experts.122.down_proj", "model.layers.25.mlp.experts.123.down_proj", "model.layers.25.mlp.experts.124.down_proj", "model.layers.25.mlp.experts.125.down_proj", "model.layers.25.mlp.experts.126.down_proj", "model.layers.25.mlp.experts.127.down_proj", "model.layers.25.mlp.experts.128.down_proj", "model.layers.25.mlp.experts.129.down_proj", "model.layers.25.mlp.experts.130.down_proj", "model.layers.25.mlp.experts.131.down_proj", "model.layers.25.mlp.experts.132.down_proj", "model.layers.25.mlp.experts.133.down_proj", "model.layers.25.mlp.experts.134.down_proj", "model.layers.25.mlp.experts.135.down_proj", "model.layers.25.mlp.experts.136.down_proj", "model.layers.25.mlp.experts.137.down_proj", "model.layers.25.mlp.experts.138.down_proj", "model.layers.25.mlp.experts.139.down_proj", "model.layers.25.mlp.experts.140.down_proj", "model.layers.25.mlp.experts.141.down_proj", "model.layers.25.mlp.experts.142.down_proj", "model.layers.25.mlp.experts.143.down_proj", "model.layers.25.mlp.experts.144.down_proj", "model.layers.25.mlp.experts.145.down_proj", "model.layers.25.mlp.experts.146.down_proj", "model.layers.25.mlp.experts.147.down_proj", "model.layers.25.mlp.experts.148.down_proj", "model.layers.25.mlp.experts.149.down_proj", "model.layers.25.mlp.experts.150.down_proj", "model.layers.25.mlp.experts.151.down_proj", "model.layers.25.mlp.experts.152.down_proj", "model.layers.25.mlp.experts.153.down_proj", "model.layers.25.mlp.experts.154.down_proj", "model.layers.25.mlp.experts.155.down_proj", "model.layers.25.mlp.experts.156.down_proj", "model.layers.25.mlp.experts.157.down_proj", "model.layers.25.mlp.experts.158.down_proj", "model.layers.25.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.064509994350374e-06, "dbits": 2516582400 } ] }, { "idx": 153, "layers": [ "model.layers.26.self_attn.q_proj" ], "candidates": [ { "dkld": 6.588001269846938e-07, "dbits": 125829120 } ] }, { "idx": 154, "layers": [ "model.layers.26.self_attn.k_proj", "model.layers.26.self_attn.v_proj" ], "candidates": [ { "dkld": -5.034214700572214e-06, "dbits": 20971520 } ] }, { "idx": 155, "layers": [ "model.layers.26.self_attn.o_proj" ], "candidates": [ { "dkld": -3.372374339960472e-06, "dbits": 125829120 } ] }, { "idx": 156, "layers": [ "model.layers.26.mlp.shared_experts.gate_proj", "model.layers.26.mlp.shared_experts.up_proj", "model.layers.26.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.970244481228205e-06, "dbits": 47185920 } ] }, { "idx": 157, "layers": [ "model.layers.26.mlp.experts.0.gate_proj", "model.layers.26.mlp.experts.1.gate_proj", "model.layers.26.mlp.experts.2.gate_proj", "model.layers.26.mlp.experts.3.gate_proj", "model.layers.26.mlp.experts.4.gate_proj", "model.layers.26.mlp.experts.5.gate_proj", "model.layers.26.mlp.experts.6.gate_proj", "model.layers.26.mlp.experts.7.gate_proj", "model.layers.26.mlp.experts.8.gate_proj", "model.layers.26.mlp.experts.9.gate_proj", "model.layers.26.mlp.experts.10.gate_proj", "model.layers.26.mlp.experts.11.gate_proj", "model.layers.26.mlp.experts.12.gate_proj", "model.layers.26.mlp.experts.13.gate_proj", "model.layers.26.mlp.experts.14.gate_proj", "model.layers.26.mlp.experts.15.gate_proj", "model.layers.26.mlp.experts.16.gate_proj", "model.layers.26.mlp.experts.17.gate_proj", "model.layers.26.mlp.experts.18.gate_proj", "model.layers.26.mlp.experts.19.gate_proj", "model.layers.26.mlp.experts.20.gate_proj", "model.layers.26.mlp.experts.21.gate_proj", "model.layers.26.mlp.experts.22.gate_proj", "model.layers.26.mlp.experts.23.gate_proj", "model.layers.26.mlp.experts.24.gate_proj", "model.layers.26.mlp.experts.25.gate_proj", "model.layers.26.mlp.experts.26.gate_proj", "model.layers.26.mlp.experts.27.gate_proj", "model.layers.26.mlp.experts.28.gate_proj", "model.layers.26.mlp.experts.29.gate_proj", "model.layers.26.mlp.experts.30.gate_proj", "model.layers.26.mlp.experts.31.gate_proj", "model.layers.26.mlp.experts.32.gate_proj", "model.layers.26.mlp.experts.33.gate_proj", "model.layers.26.mlp.experts.34.gate_proj", "model.layers.26.mlp.experts.35.gate_proj", "model.layers.26.mlp.experts.36.gate_proj", "model.layers.26.mlp.experts.37.gate_proj", "model.layers.26.mlp.experts.38.gate_proj", "model.layers.26.mlp.experts.39.gate_proj", "model.layers.26.mlp.experts.40.gate_proj", "model.layers.26.mlp.experts.41.gate_proj", "model.layers.26.mlp.experts.42.gate_proj", "model.layers.26.mlp.experts.43.gate_proj", "model.layers.26.mlp.experts.44.gate_proj", "model.layers.26.mlp.experts.45.gate_proj", "model.layers.26.mlp.experts.46.gate_proj", "model.layers.26.mlp.experts.47.gate_proj", "model.layers.26.mlp.experts.48.gate_proj", "model.layers.26.mlp.experts.49.gate_proj", "model.layers.26.mlp.experts.50.gate_proj", "model.layers.26.mlp.experts.51.gate_proj", "model.layers.26.mlp.experts.52.gate_proj", "model.layers.26.mlp.experts.53.gate_proj", "model.layers.26.mlp.experts.54.gate_proj", "model.layers.26.mlp.experts.55.gate_proj", "model.layers.26.mlp.experts.56.gate_proj", "model.layers.26.mlp.experts.57.gate_proj", "model.layers.26.mlp.experts.58.gate_proj", "model.layers.26.mlp.experts.59.gate_proj", "model.layers.26.mlp.experts.60.gate_proj", "model.layers.26.mlp.experts.61.gate_proj", "model.layers.26.mlp.experts.62.gate_proj", "model.layers.26.mlp.experts.63.gate_proj", "model.layers.26.mlp.experts.64.gate_proj", "model.layers.26.mlp.experts.65.gate_proj", "model.layers.26.mlp.experts.66.gate_proj", "model.layers.26.mlp.experts.67.gate_proj", "model.layers.26.mlp.experts.68.gate_proj", "model.layers.26.mlp.experts.69.gate_proj", "model.layers.26.mlp.experts.70.gate_proj", "model.layers.26.mlp.experts.71.gate_proj", "model.layers.26.mlp.experts.72.gate_proj", "model.layers.26.mlp.experts.73.gate_proj", "model.layers.26.mlp.experts.74.gate_proj", "model.layers.26.mlp.experts.75.gate_proj", "model.layers.26.mlp.experts.76.gate_proj", "model.layers.26.mlp.experts.77.gate_proj", "model.layers.26.mlp.experts.78.gate_proj", "model.layers.26.mlp.experts.79.gate_proj", "model.layers.26.mlp.experts.80.gate_proj", "model.layers.26.mlp.experts.81.gate_proj", "model.layers.26.mlp.experts.82.gate_proj", "model.layers.26.mlp.experts.83.gate_proj", "model.layers.26.mlp.experts.84.gate_proj", "model.layers.26.mlp.experts.85.gate_proj", "model.layers.26.mlp.experts.86.gate_proj", "model.layers.26.mlp.experts.87.gate_proj", "model.layers.26.mlp.experts.88.gate_proj", "model.layers.26.mlp.experts.89.gate_proj", "model.layers.26.mlp.experts.90.gate_proj", "model.layers.26.mlp.experts.91.gate_proj", "model.layers.26.mlp.experts.92.gate_proj", "model.layers.26.mlp.experts.93.gate_proj", "model.layers.26.mlp.experts.94.gate_proj", "model.layers.26.mlp.experts.95.gate_proj", "model.layers.26.mlp.experts.96.gate_proj", "model.layers.26.mlp.experts.97.gate_proj", "model.layers.26.mlp.experts.98.gate_proj", "model.layers.26.mlp.experts.99.gate_proj", "model.layers.26.mlp.experts.100.gate_proj", "model.layers.26.mlp.experts.101.gate_proj", "model.layers.26.mlp.experts.102.gate_proj", "model.layers.26.mlp.experts.103.gate_proj", "model.layers.26.mlp.experts.104.gate_proj", "model.layers.26.mlp.experts.105.gate_proj", "model.layers.26.mlp.experts.106.gate_proj", "model.layers.26.mlp.experts.107.gate_proj", "model.layers.26.mlp.experts.108.gate_proj", "model.layers.26.mlp.experts.109.gate_proj", "model.layers.26.mlp.experts.110.gate_proj", "model.layers.26.mlp.experts.111.gate_proj", "model.layers.26.mlp.experts.112.gate_proj", "model.layers.26.mlp.experts.113.gate_proj", "model.layers.26.mlp.experts.114.gate_proj", "model.layers.26.mlp.experts.115.gate_proj", "model.layers.26.mlp.experts.116.gate_proj", "model.layers.26.mlp.experts.117.gate_proj", "model.layers.26.mlp.experts.118.gate_proj", "model.layers.26.mlp.experts.119.gate_proj", "model.layers.26.mlp.experts.120.gate_proj", "model.layers.26.mlp.experts.121.gate_proj", "model.layers.26.mlp.experts.122.gate_proj", "model.layers.26.mlp.experts.123.gate_proj", "model.layers.26.mlp.experts.124.gate_proj", "model.layers.26.mlp.experts.125.gate_proj", "model.layers.26.mlp.experts.126.gate_proj", "model.layers.26.mlp.experts.127.gate_proj", "model.layers.26.mlp.experts.128.gate_proj", "model.layers.26.mlp.experts.129.gate_proj", "model.layers.26.mlp.experts.130.gate_proj", "model.layers.26.mlp.experts.131.gate_proj", "model.layers.26.mlp.experts.132.gate_proj", "model.layers.26.mlp.experts.133.gate_proj", "model.layers.26.mlp.experts.134.gate_proj", "model.layers.26.mlp.experts.135.gate_proj", "model.layers.26.mlp.experts.136.gate_proj", "model.layers.26.mlp.experts.137.gate_proj", "model.layers.26.mlp.experts.138.gate_proj", "model.layers.26.mlp.experts.139.gate_proj", "model.layers.26.mlp.experts.140.gate_proj", "model.layers.26.mlp.experts.141.gate_proj", "model.layers.26.mlp.experts.142.gate_proj", "model.layers.26.mlp.experts.143.gate_proj", "model.layers.26.mlp.experts.144.gate_proj", "model.layers.26.mlp.experts.145.gate_proj", "model.layers.26.mlp.experts.146.gate_proj", "model.layers.26.mlp.experts.147.gate_proj", "model.layers.26.mlp.experts.148.gate_proj", "model.layers.26.mlp.experts.149.gate_proj", "model.layers.26.mlp.experts.150.gate_proj", "model.layers.26.mlp.experts.151.gate_proj", "model.layers.26.mlp.experts.152.gate_proj", "model.layers.26.mlp.experts.153.gate_proj", "model.layers.26.mlp.experts.154.gate_proj", "model.layers.26.mlp.experts.155.gate_proj", "model.layers.26.mlp.experts.156.gate_proj", "model.layers.26.mlp.experts.157.gate_proj", "model.layers.26.mlp.experts.158.gate_proj", "model.layers.26.mlp.experts.159.gate_proj", "model.layers.26.mlp.experts.0.up_proj", "model.layers.26.mlp.experts.1.up_proj", "model.layers.26.mlp.experts.2.up_proj", "model.layers.26.mlp.experts.3.up_proj", "model.layers.26.mlp.experts.4.up_proj", "model.layers.26.mlp.experts.5.up_proj", "model.layers.26.mlp.experts.6.up_proj", "model.layers.26.mlp.experts.7.up_proj", "model.layers.26.mlp.experts.8.up_proj", "model.layers.26.mlp.experts.9.up_proj", "model.layers.26.mlp.experts.10.up_proj", "model.layers.26.mlp.experts.11.up_proj", "model.layers.26.mlp.experts.12.up_proj", "model.layers.26.mlp.experts.13.up_proj", "model.layers.26.mlp.experts.14.up_proj", "model.layers.26.mlp.experts.15.up_proj", "model.layers.26.mlp.experts.16.up_proj", "model.layers.26.mlp.experts.17.up_proj", "model.layers.26.mlp.experts.18.up_proj", "model.layers.26.mlp.experts.19.up_proj", "model.layers.26.mlp.experts.20.up_proj", "model.layers.26.mlp.experts.21.up_proj", "model.layers.26.mlp.experts.22.up_proj", "model.layers.26.mlp.experts.23.up_proj", "model.layers.26.mlp.experts.24.up_proj", "model.layers.26.mlp.experts.25.up_proj", "model.layers.26.mlp.experts.26.up_proj", "model.layers.26.mlp.experts.27.up_proj", "model.layers.26.mlp.experts.28.up_proj", "model.layers.26.mlp.experts.29.up_proj", "model.layers.26.mlp.experts.30.up_proj", "model.layers.26.mlp.experts.31.up_proj", "model.layers.26.mlp.experts.32.up_proj", "model.layers.26.mlp.experts.33.up_proj", "model.layers.26.mlp.experts.34.up_proj", "model.layers.26.mlp.experts.35.up_proj", "model.layers.26.mlp.experts.36.up_proj", "model.layers.26.mlp.experts.37.up_proj", "model.layers.26.mlp.experts.38.up_proj", "model.layers.26.mlp.experts.39.up_proj", "model.layers.26.mlp.experts.40.up_proj", "model.layers.26.mlp.experts.41.up_proj", "model.layers.26.mlp.experts.42.up_proj", "model.layers.26.mlp.experts.43.up_proj", "model.layers.26.mlp.experts.44.up_proj", "model.layers.26.mlp.experts.45.up_proj", "model.layers.26.mlp.experts.46.up_proj", "model.layers.26.mlp.experts.47.up_proj", "model.layers.26.mlp.experts.48.up_proj", "model.layers.26.mlp.experts.49.up_proj", "model.layers.26.mlp.experts.50.up_proj", "model.layers.26.mlp.experts.51.up_proj", "model.layers.26.mlp.experts.52.up_proj", "model.layers.26.mlp.experts.53.up_proj", "model.layers.26.mlp.experts.54.up_proj", "model.layers.26.mlp.experts.55.up_proj", "model.layers.26.mlp.experts.56.up_proj", "model.layers.26.mlp.experts.57.up_proj", "model.layers.26.mlp.experts.58.up_proj", "model.layers.26.mlp.experts.59.up_proj", "model.layers.26.mlp.experts.60.up_proj", "model.layers.26.mlp.experts.61.up_proj", "model.layers.26.mlp.experts.62.up_proj", "model.layers.26.mlp.experts.63.up_proj", "model.layers.26.mlp.experts.64.up_proj", "model.layers.26.mlp.experts.65.up_proj", "model.layers.26.mlp.experts.66.up_proj", "model.layers.26.mlp.experts.67.up_proj", "model.layers.26.mlp.experts.68.up_proj", "model.layers.26.mlp.experts.69.up_proj", "model.layers.26.mlp.experts.70.up_proj", "model.layers.26.mlp.experts.71.up_proj", "model.layers.26.mlp.experts.72.up_proj", "model.layers.26.mlp.experts.73.up_proj", "model.layers.26.mlp.experts.74.up_proj", "model.layers.26.mlp.experts.75.up_proj", "model.layers.26.mlp.experts.76.up_proj", "model.layers.26.mlp.experts.77.up_proj", "model.layers.26.mlp.experts.78.up_proj", "model.layers.26.mlp.experts.79.up_proj", "model.layers.26.mlp.experts.80.up_proj", "model.layers.26.mlp.experts.81.up_proj", "model.layers.26.mlp.experts.82.up_proj", "model.layers.26.mlp.experts.83.up_proj", "model.layers.26.mlp.experts.84.up_proj", "model.layers.26.mlp.experts.85.up_proj", "model.layers.26.mlp.experts.86.up_proj", "model.layers.26.mlp.experts.87.up_proj", "model.layers.26.mlp.experts.88.up_proj", "model.layers.26.mlp.experts.89.up_proj", "model.layers.26.mlp.experts.90.up_proj", "model.layers.26.mlp.experts.91.up_proj", "model.layers.26.mlp.experts.92.up_proj", "model.layers.26.mlp.experts.93.up_proj", "model.layers.26.mlp.experts.94.up_proj", "model.layers.26.mlp.experts.95.up_proj", "model.layers.26.mlp.experts.96.up_proj", "model.layers.26.mlp.experts.97.up_proj", "model.layers.26.mlp.experts.98.up_proj", "model.layers.26.mlp.experts.99.up_proj", "model.layers.26.mlp.experts.100.up_proj", "model.layers.26.mlp.experts.101.up_proj", "model.layers.26.mlp.experts.102.up_proj", "model.layers.26.mlp.experts.103.up_proj", "model.layers.26.mlp.experts.104.up_proj", "model.layers.26.mlp.experts.105.up_proj", "model.layers.26.mlp.experts.106.up_proj", "model.layers.26.mlp.experts.107.up_proj", "model.layers.26.mlp.experts.108.up_proj", "model.layers.26.mlp.experts.109.up_proj", "model.layers.26.mlp.experts.110.up_proj", "model.layers.26.mlp.experts.111.up_proj", "model.layers.26.mlp.experts.112.up_proj", "model.layers.26.mlp.experts.113.up_proj", "model.layers.26.mlp.experts.114.up_proj", "model.layers.26.mlp.experts.115.up_proj", "model.layers.26.mlp.experts.116.up_proj", "model.layers.26.mlp.experts.117.up_proj", "model.layers.26.mlp.experts.118.up_proj", "model.layers.26.mlp.experts.119.up_proj", "model.layers.26.mlp.experts.120.up_proj", "model.layers.26.mlp.experts.121.up_proj", "model.layers.26.mlp.experts.122.up_proj", "model.layers.26.mlp.experts.123.up_proj", "model.layers.26.mlp.experts.124.up_proj", "model.layers.26.mlp.experts.125.up_proj", "model.layers.26.mlp.experts.126.up_proj", "model.layers.26.mlp.experts.127.up_proj", "model.layers.26.mlp.experts.128.up_proj", "model.layers.26.mlp.experts.129.up_proj", "model.layers.26.mlp.experts.130.up_proj", "model.layers.26.mlp.experts.131.up_proj", "model.layers.26.mlp.experts.132.up_proj", "model.layers.26.mlp.experts.133.up_proj", "model.layers.26.mlp.experts.134.up_proj", "model.layers.26.mlp.experts.135.up_proj", "model.layers.26.mlp.experts.136.up_proj", "model.layers.26.mlp.experts.137.up_proj", "model.layers.26.mlp.experts.138.up_proj", "model.layers.26.mlp.experts.139.up_proj", "model.layers.26.mlp.experts.140.up_proj", "model.layers.26.mlp.experts.141.up_proj", "model.layers.26.mlp.experts.142.up_proj", "model.layers.26.mlp.experts.143.up_proj", "model.layers.26.mlp.experts.144.up_proj", "model.layers.26.mlp.experts.145.up_proj", "model.layers.26.mlp.experts.146.up_proj", "model.layers.26.mlp.experts.147.up_proj", "model.layers.26.mlp.experts.148.up_proj", "model.layers.26.mlp.experts.149.up_proj", "model.layers.26.mlp.experts.150.up_proj", "model.layers.26.mlp.experts.151.up_proj", "model.layers.26.mlp.experts.152.up_proj", "model.layers.26.mlp.experts.153.up_proj", "model.layers.26.mlp.experts.154.up_proj", "model.layers.26.mlp.experts.155.up_proj", "model.layers.26.mlp.experts.156.up_proj", "model.layers.26.mlp.experts.157.up_proj", "model.layers.26.mlp.experts.158.up_proj", "model.layers.26.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.826160147786184e-06, "dbits": 5033164800 } ] }, { "idx": 158, "layers": [ "model.layers.26.mlp.experts.0.down_proj", "model.layers.26.mlp.experts.1.down_proj", "model.layers.26.mlp.experts.2.down_proj", "model.layers.26.mlp.experts.3.down_proj", "model.layers.26.mlp.experts.4.down_proj", "model.layers.26.mlp.experts.5.down_proj", "model.layers.26.mlp.experts.6.down_proj", "model.layers.26.mlp.experts.7.down_proj", "model.layers.26.mlp.experts.8.down_proj", "model.layers.26.mlp.experts.9.down_proj", "model.layers.26.mlp.experts.10.down_proj", "model.layers.26.mlp.experts.11.down_proj", "model.layers.26.mlp.experts.12.down_proj", "model.layers.26.mlp.experts.13.down_proj", "model.layers.26.mlp.experts.14.down_proj", "model.layers.26.mlp.experts.15.down_proj", "model.layers.26.mlp.experts.16.down_proj", "model.layers.26.mlp.experts.17.down_proj", "model.layers.26.mlp.experts.18.down_proj", "model.layers.26.mlp.experts.19.down_proj", "model.layers.26.mlp.experts.20.down_proj", "model.layers.26.mlp.experts.21.down_proj", "model.layers.26.mlp.experts.22.down_proj", "model.layers.26.mlp.experts.23.down_proj", "model.layers.26.mlp.experts.24.down_proj", "model.layers.26.mlp.experts.25.down_proj", "model.layers.26.mlp.experts.26.down_proj", "model.layers.26.mlp.experts.27.down_proj", "model.layers.26.mlp.experts.28.down_proj", "model.layers.26.mlp.experts.29.down_proj", "model.layers.26.mlp.experts.30.down_proj", "model.layers.26.mlp.experts.31.down_proj", "model.layers.26.mlp.experts.32.down_proj", "model.layers.26.mlp.experts.33.down_proj", "model.layers.26.mlp.experts.34.down_proj", "model.layers.26.mlp.experts.35.down_proj", "model.layers.26.mlp.experts.36.down_proj", "model.layers.26.mlp.experts.37.down_proj", "model.layers.26.mlp.experts.38.down_proj", "model.layers.26.mlp.experts.39.down_proj", "model.layers.26.mlp.experts.40.down_proj", "model.layers.26.mlp.experts.41.down_proj", "model.layers.26.mlp.experts.42.down_proj", "model.layers.26.mlp.experts.43.down_proj", "model.layers.26.mlp.experts.44.down_proj", "model.layers.26.mlp.experts.45.down_proj", "model.layers.26.mlp.experts.46.down_proj", "model.layers.26.mlp.experts.47.down_proj", "model.layers.26.mlp.experts.48.down_proj", "model.layers.26.mlp.experts.49.down_proj", "model.layers.26.mlp.experts.50.down_proj", "model.layers.26.mlp.experts.51.down_proj", "model.layers.26.mlp.experts.52.down_proj", "model.layers.26.mlp.experts.53.down_proj", "model.layers.26.mlp.experts.54.down_proj", "model.layers.26.mlp.experts.55.down_proj", "model.layers.26.mlp.experts.56.down_proj", "model.layers.26.mlp.experts.57.down_proj", "model.layers.26.mlp.experts.58.down_proj", "model.layers.26.mlp.experts.59.down_proj", "model.layers.26.mlp.experts.60.down_proj", "model.layers.26.mlp.experts.61.down_proj", "model.layers.26.mlp.experts.62.down_proj", "model.layers.26.mlp.experts.63.down_proj", "model.layers.26.mlp.experts.64.down_proj", "model.layers.26.mlp.experts.65.down_proj", "model.layers.26.mlp.experts.66.down_proj", "model.layers.26.mlp.experts.67.down_proj", "model.layers.26.mlp.experts.68.down_proj", "model.layers.26.mlp.experts.69.down_proj", "model.layers.26.mlp.experts.70.down_proj", "model.layers.26.mlp.experts.71.down_proj", "model.layers.26.mlp.experts.72.down_proj", "model.layers.26.mlp.experts.73.down_proj", "model.layers.26.mlp.experts.74.down_proj", "model.layers.26.mlp.experts.75.down_proj", "model.layers.26.mlp.experts.76.down_proj", "model.layers.26.mlp.experts.77.down_proj", "model.layers.26.mlp.experts.78.down_proj", "model.layers.26.mlp.experts.79.down_proj", "model.layers.26.mlp.experts.80.down_proj", "model.layers.26.mlp.experts.81.down_proj", "model.layers.26.mlp.experts.82.down_proj", "model.layers.26.mlp.experts.83.down_proj", "model.layers.26.mlp.experts.84.down_proj", "model.layers.26.mlp.experts.85.down_proj", "model.layers.26.mlp.experts.86.down_proj", "model.layers.26.mlp.experts.87.down_proj", "model.layers.26.mlp.experts.88.down_proj", "model.layers.26.mlp.experts.89.down_proj", "model.layers.26.mlp.experts.90.down_proj", "model.layers.26.mlp.experts.91.down_proj", "model.layers.26.mlp.experts.92.down_proj", "model.layers.26.mlp.experts.93.down_proj", "model.layers.26.mlp.experts.94.down_proj", "model.layers.26.mlp.experts.95.down_proj", "model.layers.26.mlp.experts.96.down_proj", "model.layers.26.mlp.experts.97.down_proj", "model.layers.26.mlp.experts.98.down_proj", "model.layers.26.mlp.experts.99.down_proj", "model.layers.26.mlp.experts.100.down_proj", "model.layers.26.mlp.experts.101.down_proj", "model.layers.26.mlp.experts.102.down_proj", "model.layers.26.mlp.experts.103.down_proj", "model.layers.26.mlp.experts.104.down_proj", "model.layers.26.mlp.experts.105.down_proj", "model.layers.26.mlp.experts.106.down_proj", "model.layers.26.mlp.experts.107.down_proj", "model.layers.26.mlp.experts.108.down_proj", "model.layers.26.mlp.experts.109.down_proj", "model.layers.26.mlp.experts.110.down_proj", "model.layers.26.mlp.experts.111.down_proj", "model.layers.26.mlp.experts.112.down_proj", "model.layers.26.mlp.experts.113.down_proj", "model.layers.26.mlp.experts.114.down_proj", "model.layers.26.mlp.experts.115.down_proj", "model.layers.26.mlp.experts.116.down_proj", "model.layers.26.mlp.experts.117.down_proj", "model.layers.26.mlp.experts.118.down_proj", "model.layers.26.mlp.experts.119.down_proj", "model.layers.26.mlp.experts.120.down_proj", "model.layers.26.mlp.experts.121.down_proj", "model.layers.26.mlp.experts.122.down_proj", "model.layers.26.mlp.experts.123.down_proj", "model.layers.26.mlp.experts.124.down_proj", "model.layers.26.mlp.experts.125.down_proj", "model.layers.26.mlp.experts.126.down_proj", "model.layers.26.mlp.experts.127.down_proj", "model.layers.26.mlp.experts.128.down_proj", "model.layers.26.mlp.experts.129.down_proj", "model.layers.26.mlp.experts.130.down_proj", "model.layers.26.mlp.experts.131.down_proj", "model.layers.26.mlp.experts.132.down_proj", "model.layers.26.mlp.experts.133.down_proj", "model.layers.26.mlp.experts.134.down_proj", "model.layers.26.mlp.experts.135.down_proj", "model.layers.26.mlp.experts.136.down_proj", "model.layers.26.mlp.experts.137.down_proj", "model.layers.26.mlp.experts.138.down_proj", "model.layers.26.mlp.experts.139.down_proj", "model.layers.26.mlp.experts.140.down_proj", "model.layers.26.mlp.experts.141.down_proj", "model.layers.26.mlp.experts.142.down_proj", "model.layers.26.mlp.experts.143.down_proj", "model.layers.26.mlp.experts.144.down_proj", "model.layers.26.mlp.experts.145.down_proj", "model.layers.26.mlp.experts.146.down_proj", "model.layers.26.mlp.experts.147.down_proj", "model.layers.26.mlp.experts.148.down_proj", "model.layers.26.mlp.experts.149.down_proj", "model.layers.26.mlp.experts.150.down_proj", "model.layers.26.mlp.experts.151.down_proj", "model.layers.26.mlp.experts.152.down_proj", "model.layers.26.mlp.experts.153.down_proj", "model.layers.26.mlp.experts.154.down_proj", "model.layers.26.mlp.experts.155.down_proj", "model.layers.26.mlp.experts.156.down_proj", "model.layers.26.mlp.experts.157.down_proj", "model.layers.26.mlp.experts.158.down_proj", "model.layers.26.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -4.287902265787103e-06, "dbits": 2516582400 } ] }, { "idx": 159, "layers": [ "model.layers.27.self_attn.q_proj" ], "candidates": [ { "dkld": -3.1742063583805723e-06, "dbits": 125829120 } ] }, { "idx": 160, "layers": [ "model.layers.27.self_attn.k_proj", "model.layers.27.self_attn.v_proj" ], "candidates": [ { "dkld": -4.809050005860605e-06, "dbits": 20971520 } ] }, { "idx": 161, "layers": [ "model.layers.27.self_attn.o_proj" ], "candidates": [ { "dkld": -5.477911327034278e-06, "dbits": 125829120 } ] }, { "idx": 162, "layers": [ "model.layers.27.mlp.shared_experts.gate_proj", "model.layers.27.mlp.shared_experts.up_proj", "model.layers.27.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 5.968072218820409e-06, "dbits": 47185920 } ] }, { "idx": 163, "layers": [ "model.layers.27.mlp.experts.0.gate_proj", "model.layers.27.mlp.experts.1.gate_proj", "model.layers.27.mlp.experts.2.gate_proj", "model.layers.27.mlp.experts.3.gate_proj", "model.layers.27.mlp.experts.4.gate_proj", "model.layers.27.mlp.experts.5.gate_proj", "model.layers.27.mlp.experts.6.gate_proj", "model.layers.27.mlp.experts.7.gate_proj", "model.layers.27.mlp.experts.8.gate_proj", "model.layers.27.mlp.experts.9.gate_proj", "model.layers.27.mlp.experts.10.gate_proj", "model.layers.27.mlp.experts.11.gate_proj", "model.layers.27.mlp.experts.12.gate_proj", "model.layers.27.mlp.experts.13.gate_proj", "model.layers.27.mlp.experts.14.gate_proj", "model.layers.27.mlp.experts.15.gate_proj", "model.layers.27.mlp.experts.16.gate_proj", "model.layers.27.mlp.experts.17.gate_proj", "model.layers.27.mlp.experts.18.gate_proj", "model.layers.27.mlp.experts.19.gate_proj", "model.layers.27.mlp.experts.20.gate_proj", "model.layers.27.mlp.experts.21.gate_proj", "model.layers.27.mlp.experts.22.gate_proj", "model.layers.27.mlp.experts.23.gate_proj", "model.layers.27.mlp.experts.24.gate_proj", "model.layers.27.mlp.experts.25.gate_proj", "model.layers.27.mlp.experts.26.gate_proj", "model.layers.27.mlp.experts.27.gate_proj", "model.layers.27.mlp.experts.28.gate_proj", "model.layers.27.mlp.experts.29.gate_proj", "model.layers.27.mlp.experts.30.gate_proj", "model.layers.27.mlp.experts.31.gate_proj", "model.layers.27.mlp.experts.32.gate_proj", "model.layers.27.mlp.experts.33.gate_proj", "model.layers.27.mlp.experts.34.gate_proj", "model.layers.27.mlp.experts.35.gate_proj", "model.layers.27.mlp.experts.36.gate_proj", "model.layers.27.mlp.experts.37.gate_proj", "model.layers.27.mlp.experts.38.gate_proj", "model.layers.27.mlp.experts.39.gate_proj", "model.layers.27.mlp.experts.40.gate_proj", "model.layers.27.mlp.experts.41.gate_proj", "model.layers.27.mlp.experts.42.gate_proj", "model.layers.27.mlp.experts.43.gate_proj", "model.layers.27.mlp.experts.44.gate_proj", "model.layers.27.mlp.experts.45.gate_proj", "model.layers.27.mlp.experts.46.gate_proj", "model.layers.27.mlp.experts.47.gate_proj", "model.layers.27.mlp.experts.48.gate_proj", "model.layers.27.mlp.experts.49.gate_proj", "model.layers.27.mlp.experts.50.gate_proj", "model.layers.27.mlp.experts.51.gate_proj", "model.layers.27.mlp.experts.52.gate_proj", "model.layers.27.mlp.experts.53.gate_proj", "model.layers.27.mlp.experts.54.gate_proj", "model.layers.27.mlp.experts.55.gate_proj", "model.layers.27.mlp.experts.56.gate_proj", "model.layers.27.mlp.experts.57.gate_proj", "model.layers.27.mlp.experts.58.gate_proj", "model.layers.27.mlp.experts.59.gate_proj", "model.layers.27.mlp.experts.60.gate_proj", "model.layers.27.mlp.experts.61.gate_proj", "model.layers.27.mlp.experts.62.gate_proj", "model.layers.27.mlp.experts.63.gate_proj", "model.layers.27.mlp.experts.64.gate_proj", "model.layers.27.mlp.experts.65.gate_proj", "model.layers.27.mlp.experts.66.gate_proj", "model.layers.27.mlp.experts.67.gate_proj", "model.layers.27.mlp.experts.68.gate_proj", "model.layers.27.mlp.experts.69.gate_proj", "model.layers.27.mlp.experts.70.gate_proj", "model.layers.27.mlp.experts.71.gate_proj", "model.layers.27.mlp.experts.72.gate_proj", "model.layers.27.mlp.experts.73.gate_proj", "model.layers.27.mlp.experts.74.gate_proj", "model.layers.27.mlp.experts.75.gate_proj", "model.layers.27.mlp.experts.76.gate_proj", "model.layers.27.mlp.experts.77.gate_proj", "model.layers.27.mlp.experts.78.gate_proj", "model.layers.27.mlp.experts.79.gate_proj", "model.layers.27.mlp.experts.80.gate_proj", "model.layers.27.mlp.experts.81.gate_proj", "model.layers.27.mlp.experts.82.gate_proj", "model.layers.27.mlp.experts.83.gate_proj", "model.layers.27.mlp.experts.84.gate_proj", "model.layers.27.mlp.experts.85.gate_proj", "model.layers.27.mlp.experts.86.gate_proj", "model.layers.27.mlp.experts.87.gate_proj", "model.layers.27.mlp.experts.88.gate_proj", "model.layers.27.mlp.experts.89.gate_proj", "model.layers.27.mlp.experts.90.gate_proj", "model.layers.27.mlp.experts.91.gate_proj", "model.layers.27.mlp.experts.92.gate_proj", "model.layers.27.mlp.experts.93.gate_proj", "model.layers.27.mlp.experts.94.gate_proj", "model.layers.27.mlp.experts.95.gate_proj", "model.layers.27.mlp.experts.96.gate_proj", "model.layers.27.mlp.experts.97.gate_proj", "model.layers.27.mlp.experts.98.gate_proj", "model.layers.27.mlp.experts.99.gate_proj", "model.layers.27.mlp.experts.100.gate_proj", "model.layers.27.mlp.experts.101.gate_proj", "model.layers.27.mlp.experts.102.gate_proj", "model.layers.27.mlp.experts.103.gate_proj", "model.layers.27.mlp.experts.104.gate_proj", "model.layers.27.mlp.experts.105.gate_proj", "model.layers.27.mlp.experts.106.gate_proj", "model.layers.27.mlp.experts.107.gate_proj", "model.layers.27.mlp.experts.108.gate_proj", "model.layers.27.mlp.experts.109.gate_proj", "model.layers.27.mlp.experts.110.gate_proj", "model.layers.27.mlp.experts.111.gate_proj", "model.layers.27.mlp.experts.112.gate_proj", "model.layers.27.mlp.experts.113.gate_proj", "model.layers.27.mlp.experts.114.gate_proj", "model.layers.27.mlp.experts.115.gate_proj", "model.layers.27.mlp.experts.116.gate_proj", "model.layers.27.mlp.experts.117.gate_proj", "model.layers.27.mlp.experts.118.gate_proj", "model.layers.27.mlp.experts.119.gate_proj", "model.layers.27.mlp.experts.120.gate_proj", "model.layers.27.mlp.experts.121.gate_proj", "model.layers.27.mlp.experts.122.gate_proj", "model.layers.27.mlp.experts.123.gate_proj", "model.layers.27.mlp.experts.124.gate_proj", "model.layers.27.mlp.experts.125.gate_proj", "model.layers.27.mlp.experts.126.gate_proj", "model.layers.27.mlp.experts.127.gate_proj", "model.layers.27.mlp.experts.128.gate_proj", "model.layers.27.mlp.experts.129.gate_proj", "model.layers.27.mlp.experts.130.gate_proj", "model.layers.27.mlp.experts.131.gate_proj", "model.layers.27.mlp.experts.132.gate_proj", "model.layers.27.mlp.experts.133.gate_proj", "model.layers.27.mlp.experts.134.gate_proj", "model.layers.27.mlp.experts.135.gate_proj", "model.layers.27.mlp.experts.136.gate_proj", "model.layers.27.mlp.experts.137.gate_proj", "model.layers.27.mlp.experts.138.gate_proj", "model.layers.27.mlp.experts.139.gate_proj", "model.layers.27.mlp.experts.140.gate_proj", "model.layers.27.mlp.experts.141.gate_proj", "model.layers.27.mlp.experts.142.gate_proj", "model.layers.27.mlp.experts.143.gate_proj", "model.layers.27.mlp.experts.144.gate_proj", "model.layers.27.mlp.experts.145.gate_proj", "model.layers.27.mlp.experts.146.gate_proj", "model.layers.27.mlp.experts.147.gate_proj", "model.layers.27.mlp.experts.148.gate_proj", "model.layers.27.mlp.experts.149.gate_proj", "model.layers.27.mlp.experts.150.gate_proj", "model.layers.27.mlp.experts.151.gate_proj", "model.layers.27.mlp.experts.152.gate_proj", "model.layers.27.mlp.experts.153.gate_proj", "model.layers.27.mlp.experts.154.gate_proj", "model.layers.27.mlp.experts.155.gate_proj", "model.layers.27.mlp.experts.156.gate_proj", "model.layers.27.mlp.experts.157.gate_proj", "model.layers.27.mlp.experts.158.gate_proj", "model.layers.27.mlp.experts.159.gate_proj", "model.layers.27.mlp.experts.0.up_proj", "model.layers.27.mlp.experts.1.up_proj", "model.layers.27.mlp.experts.2.up_proj", "model.layers.27.mlp.experts.3.up_proj", "model.layers.27.mlp.experts.4.up_proj", "model.layers.27.mlp.experts.5.up_proj", "model.layers.27.mlp.experts.6.up_proj", "model.layers.27.mlp.experts.7.up_proj", "model.layers.27.mlp.experts.8.up_proj", "model.layers.27.mlp.experts.9.up_proj", "model.layers.27.mlp.experts.10.up_proj", "model.layers.27.mlp.experts.11.up_proj", "model.layers.27.mlp.experts.12.up_proj", "model.layers.27.mlp.experts.13.up_proj", "model.layers.27.mlp.experts.14.up_proj", "model.layers.27.mlp.experts.15.up_proj", "model.layers.27.mlp.experts.16.up_proj", "model.layers.27.mlp.experts.17.up_proj", "model.layers.27.mlp.experts.18.up_proj", "model.layers.27.mlp.experts.19.up_proj", "model.layers.27.mlp.experts.20.up_proj", "model.layers.27.mlp.experts.21.up_proj", "model.layers.27.mlp.experts.22.up_proj", "model.layers.27.mlp.experts.23.up_proj", "model.layers.27.mlp.experts.24.up_proj", "model.layers.27.mlp.experts.25.up_proj", "model.layers.27.mlp.experts.26.up_proj", "model.layers.27.mlp.experts.27.up_proj", "model.layers.27.mlp.experts.28.up_proj", "model.layers.27.mlp.experts.29.up_proj", "model.layers.27.mlp.experts.30.up_proj", "model.layers.27.mlp.experts.31.up_proj", "model.layers.27.mlp.experts.32.up_proj", "model.layers.27.mlp.experts.33.up_proj", "model.layers.27.mlp.experts.34.up_proj", "model.layers.27.mlp.experts.35.up_proj", "model.layers.27.mlp.experts.36.up_proj", "model.layers.27.mlp.experts.37.up_proj", "model.layers.27.mlp.experts.38.up_proj", "model.layers.27.mlp.experts.39.up_proj", "model.layers.27.mlp.experts.40.up_proj", "model.layers.27.mlp.experts.41.up_proj", "model.layers.27.mlp.experts.42.up_proj", "model.layers.27.mlp.experts.43.up_proj", "model.layers.27.mlp.experts.44.up_proj", "model.layers.27.mlp.experts.45.up_proj", "model.layers.27.mlp.experts.46.up_proj", "model.layers.27.mlp.experts.47.up_proj", "model.layers.27.mlp.experts.48.up_proj", "model.layers.27.mlp.experts.49.up_proj", "model.layers.27.mlp.experts.50.up_proj", "model.layers.27.mlp.experts.51.up_proj", "model.layers.27.mlp.experts.52.up_proj", "model.layers.27.mlp.experts.53.up_proj", "model.layers.27.mlp.experts.54.up_proj", "model.layers.27.mlp.experts.55.up_proj", "model.layers.27.mlp.experts.56.up_proj", "model.layers.27.mlp.experts.57.up_proj", "model.layers.27.mlp.experts.58.up_proj", "model.layers.27.mlp.experts.59.up_proj", "model.layers.27.mlp.experts.60.up_proj", "model.layers.27.mlp.experts.61.up_proj", "model.layers.27.mlp.experts.62.up_proj", "model.layers.27.mlp.experts.63.up_proj", "model.layers.27.mlp.experts.64.up_proj", "model.layers.27.mlp.experts.65.up_proj", "model.layers.27.mlp.experts.66.up_proj", "model.layers.27.mlp.experts.67.up_proj", "model.layers.27.mlp.experts.68.up_proj", "model.layers.27.mlp.experts.69.up_proj", "model.layers.27.mlp.experts.70.up_proj", "model.layers.27.mlp.experts.71.up_proj", "model.layers.27.mlp.experts.72.up_proj", "model.layers.27.mlp.experts.73.up_proj", "model.layers.27.mlp.experts.74.up_proj", "model.layers.27.mlp.experts.75.up_proj", "model.layers.27.mlp.experts.76.up_proj", "model.layers.27.mlp.experts.77.up_proj", "model.layers.27.mlp.experts.78.up_proj", "model.layers.27.mlp.experts.79.up_proj", "model.layers.27.mlp.experts.80.up_proj", "model.layers.27.mlp.experts.81.up_proj", "model.layers.27.mlp.experts.82.up_proj", "model.layers.27.mlp.experts.83.up_proj", "model.layers.27.mlp.experts.84.up_proj", "model.layers.27.mlp.experts.85.up_proj", "model.layers.27.mlp.experts.86.up_proj", "model.layers.27.mlp.experts.87.up_proj", "model.layers.27.mlp.experts.88.up_proj", "model.layers.27.mlp.experts.89.up_proj", "model.layers.27.mlp.experts.90.up_proj", "model.layers.27.mlp.experts.91.up_proj", "model.layers.27.mlp.experts.92.up_proj", "model.layers.27.mlp.experts.93.up_proj", "model.layers.27.mlp.experts.94.up_proj", "model.layers.27.mlp.experts.95.up_proj", "model.layers.27.mlp.experts.96.up_proj", "model.layers.27.mlp.experts.97.up_proj", "model.layers.27.mlp.experts.98.up_proj", "model.layers.27.mlp.experts.99.up_proj", "model.layers.27.mlp.experts.100.up_proj", "model.layers.27.mlp.experts.101.up_proj", "model.layers.27.mlp.experts.102.up_proj", "model.layers.27.mlp.experts.103.up_proj", "model.layers.27.mlp.experts.104.up_proj", "model.layers.27.mlp.experts.105.up_proj", "model.layers.27.mlp.experts.106.up_proj", "model.layers.27.mlp.experts.107.up_proj", "model.layers.27.mlp.experts.108.up_proj", "model.layers.27.mlp.experts.109.up_proj", "model.layers.27.mlp.experts.110.up_proj", "model.layers.27.mlp.experts.111.up_proj", "model.layers.27.mlp.experts.112.up_proj", "model.layers.27.mlp.experts.113.up_proj", "model.layers.27.mlp.experts.114.up_proj", "model.layers.27.mlp.experts.115.up_proj", "model.layers.27.mlp.experts.116.up_proj", "model.layers.27.mlp.experts.117.up_proj", "model.layers.27.mlp.experts.118.up_proj", "model.layers.27.mlp.experts.119.up_proj", "model.layers.27.mlp.experts.120.up_proj", "model.layers.27.mlp.experts.121.up_proj", "model.layers.27.mlp.experts.122.up_proj", "model.layers.27.mlp.experts.123.up_proj", "model.layers.27.mlp.experts.124.up_proj", "model.layers.27.mlp.experts.125.up_proj", "model.layers.27.mlp.experts.126.up_proj", "model.layers.27.mlp.experts.127.up_proj", "model.layers.27.mlp.experts.128.up_proj", "model.layers.27.mlp.experts.129.up_proj", "model.layers.27.mlp.experts.130.up_proj", "model.layers.27.mlp.experts.131.up_proj", "model.layers.27.mlp.experts.132.up_proj", "model.layers.27.mlp.experts.133.up_proj", "model.layers.27.mlp.experts.134.up_proj", "model.layers.27.mlp.experts.135.up_proj", "model.layers.27.mlp.experts.136.up_proj", "model.layers.27.mlp.experts.137.up_proj", "model.layers.27.mlp.experts.138.up_proj", "model.layers.27.mlp.experts.139.up_proj", "model.layers.27.mlp.experts.140.up_proj", "model.layers.27.mlp.experts.141.up_proj", "model.layers.27.mlp.experts.142.up_proj", "model.layers.27.mlp.experts.143.up_proj", "model.layers.27.mlp.experts.144.up_proj", "model.layers.27.mlp.experts.145.up_proj", "model.layers.27.mlp.experts.146.up_proj", "model.layers.27.mlp.experts.147.up_proj", "model.layers.27.mlp.experts.148.up_proj", "model.layers.27.mlp.experts.149.up_proj", "model.layers.27.mlp.experts.150.up_proj", "model.layers.27.mlp.experts.151.up_proj", "model.layers.27.mlp.experts.152.up_proj", "model.layers.27.mlp.experts.153.up_proj", "model.layers.27.mlp.experts.154.up_proj", "model.layers.27.mlp.experts.155.up_proj", "model.layers.27.mlp.experts.156.up_proj", "model.layers.27.mlp.experts.157.up_proj", "model.layers.27.mlp.experts.158.up_proj", "model.layers.27.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.023579483851747e-06, "dbits": 5033164800 } ] }, { "idx": 164, "layers": [ "model.layers.27.mlp.experts.0.down_proj", "model.layers.27.mlp.experts.1.down_proj", "model.layers.27.mlp.experts.2.down_proj", "model.layers.27.mlp.experts.3.down_proj", "model.layers.27.mlp.experts.4.down_proj", "model.layers.27.mlp.experts.5.down_proj", "model.layers.27.mlp.experts.6.down_proj", "model.layers.27.mlp.experts.7.down_proj", "model.layers.27.mlp.experts.8.down_proj", "model.layers.27.mlp.experts.9.down_proj", "model.layers.27.mlp.experts.10.down_proj", "model.layers.27.mlp.experts.11.down_proj", "model.layers.27.mlp.experts.12.down_proj", "model.layers.27.mlp.experts.13.down_proj", "model.layers.27.mlp.experts.14.down_proj", "model.layers.27.mlp.experts.15.down_proj", "model.layers.27.mlp.experts.16.down_proj", "model.layers.27.mlp.experts.17.down_proj", "model.layers.27.mlp.experts.18.down_proj", "model.layers.27.mlp.experts.19.down_proj", "model.layers.27.mlp.experts.20.down_proj", "model.layers.27.mlp.experts.21.down_proj", "model.layers.27.mlp.experts.22.down_proj", "model.layers.27.mlp.experts.23.down_proj", "model.layers.27.mlp.experts.24.down_proj", "model.layers.27.mlp.experts.25.down_proj", "model.layers.27.mlp.experts.26.down_proj", "model.layers.27.mlp.experts.27.down_proj", "model.layers.27.mlp.experts.28.down_proj", "model.layers.27.mlp.experts.29.down_proj", "model.layers.27.mlp.experts.30.down_proj", "model.layers.27.mlp.experts.31.down_proj", "model.layers.27.mlp.experts.32.down_proj", "model.layers.27.mlp.experts.33.down_proj", "model.layers.27.mlp.experts.34.down_proj", "model.layers.27.mlp.experts.35.down_proj", "model.layers.27.mlp.experts.36.down_proj", "model.layers.27.mlp.experts.37.down_proj", "model.layers.27.mlp.experts.38.down_proj", "model.layers.27.mlp.experts.39.down_proj", "model.layers.27.mlp.experts.40.down_proj", "model.layers.27.mlp.experts.41.down_proj", "model.layers.27.mlp.experts.42.down_proj", "model.layers.27.mlp.experts.43.down_proj", "model.layers.27.mlp.experts.44.down_proj", "model.layers.27.mlp.experts.45.down_proj", "model.layers.27.mlp.experts.46.down_proj", "model.layers.27.mlp.experts.47.down_proj", "model.layers.27.mlp.experts.48.down_proj", "model.layers.27.mlp.experts.49.down_proj", "model.layers.27.mlp.experts.50.down_proj", "model.layers.27.mlp.experts.51.down_proj", "model.layers.27.mlp.experts.52.down_proj", "model.layers.27.mlp.experts.53.down_proj", "model.layers.27.mlp.experts.54.down_proj", "model.layers.27.mlp.experts.55.down_proj", "model.layers.27.mlp.experts.56.down_proj", "model.layers.27.mlp.experts.57.down_proj", "model.layers.27.mlp.experts.58.down_proj", "model.layers.27.mlp.experts.59.down_proj", "model.layers.27.mlp.experts.60.down_proj", "model.layers.27.mlp.experts.61.down_proj", "model.layers.27.mlp.experts.62.down_proj", "model.layers.27.mlp.experts.63.down_proj", "model.layers.27.mlp.experts.64.down_proj", "model.layers.27.mlp.experts.65.down_proj", "model.layers.27.mlp.experts.66.down_proj", "model.layers.27.mlp.experts.67.down_proj", "model.layers.27.mlp.experts.68.down_proj", "model.layers.27.mlp.experts.69.down_proj", "model.layers.27.mlp.experts.70.down_proj", "model.layers.27.mlp.experts.71.down_proj", "model.layers.27.mlp.experts.72.down_proj", "model.layers.27.mlp.experts.73.down_proj", "model.layers.27.mlp.experts.74.down_proj", "model.layers.27.mlp.experts.75.down_proj", "model.layers.27.mlp.experts.76.down_proj", "model.layers.27.mlp.experts.77.down_proj", "model.layers.27.mlp.experts.78.down_proj", "model.layers.27.mlp.experts.79.down_proj", "model.layers.27.mlp.experts.80.down_proj", "model.layers.27.mlp.experts.81.down_proj", "model.layers.27.mlp.experts.82.down_proj", "model.layers.27.mlp.experts.83.down_proj", "model.layers.27.mlp.experts.84.down_proj", "model.layers.27.mlp.experts.85.down_proj", "model.layers.27.mlp.experts.86.down_proj", "model.layers.27.mlp.experts.87.down_proj", "model.layers.27.mlp.experts.88.down_proj", "model.layers.27.mlp.experts.89.down_proj", "model.layers.27.mlp.experts.90.down_proj", "model.layers.27.mlp.experts.91.down_proj", "model.layers.27.mlp.experts.92.down_proj", "model.layers.27.mlp.experts.93.down_proj", "model.layers.27.mlp.experts.94.down_proj", "model.layers.27.mlp.experts.95.down_proj", "model.layers.27.mlp.experts.96.down_proj", "model.layers.27.mlp.experts.97.down_proj", "model.layers.27.mlp.experts.98.down_proj", "model.layers.27.mlp.experts.99.down_proj", "model.layers.27.mlp.experts.100.down_proj", "model.layers.27.mlp.experts.101.down_proj", "model.layers.27.mlp.experts.102.down_proj", "model.layers.27.mlp.experts.103.down_proj", "model.layers.27.mlp.experts.104.down_proj", "model.layers.27.mlp.experts.105.down_proj", "model.layers.27.mlp.experts.106.down_proj", "model.layers.27.mlp.experts.107.down_proj", "model.layers.27.mlp.experts.108.down_proj", "model.layers.27.mlp.experts.109.down_proj", "model.layers.27.mlp.experts.110.down_proj", "model.layers.27.mlp.experts.111.down_proj", "model.layers.27.mlp.experts.112.down_proj", "model.layers.27.mlp.experts.113.down_proj", "model.layers.27.mlp.experts.114.down_proj", "model.layers.27.mlp.experts.115.down_proj", "model.layers.27.mlp.experts.116.down_proj", "model.layers.27.mlp.experts.117.down_proj", "model.layers.27.mlp.experts.118.down_proj", "model.layers.27.mlp.experts.119.down_proj", "model.layers.27.mlp.experts.120.down_proj", "model.layers.27.mlp.experts.121.down_proj", "model.layers.27.mlp.experts.122.down_proj", "model.layers.27.mlp.experts.123.down_proj", "model.layers.27.mlp.experts.124.down_proj", "model.layers.27.mlp.experts.125.down_proj", "model.layers.27.mlp.experts.126.down_proj", "model.layers.27.mlp.experts.127.down_proj", "model.layers.27.mlp.experts.128.down_proj", "model.layers.27.mlp.experts.129.down_proj", "model.layers.27.mlp.experts.130.down_proj", "model.layers.27.mlp.experts.131.down_proj", "model.layers.27.mlp.experts.132.down_proj", "model.layers.27.mlp.experts.133.down_proj", "model.layers.27.mlp.experts.134.down_proj", "model.layers.27.mlp.experts.135.down_proj", "model.layers.27.mlp.experts.136.down_proj", "model.layers.27.mlp.experts.137.down_proj", "model.layers.27.mlp.experts.138.down_proj", "model.layers.27.mlp.experts.139.down_proj", "model.layers.27.mlp.experts.140.down_proj", "model.layers.27.mlp.experts.141.down_proj", "model.layers.27.mlp.experts.142.down_proj", "model.layers.27.mlp.experts.143.down_proj", "model.layers.27.mlp.experts.144.down_proj", "model.layers.27.mlp.experts.145.down_proj", "model.layers.27.mlp.experts.146.down_proj", "model.layers.27.mlp.experts.147.down_proj", "model.layers.27.mlp.experts.148.down_proj", "model.layers.27.mlp.experts.149.down_proj", "model.layers.27.mlp.experts.150.down_proj", "model.layers.27.mlp.experts.151.down_proj", "model.layers.27.mlp.experts.152.down_proj", "model.layers.27.mlp.experts.153.down_proj", "model.layers.27.mlp.experts.154.down_proj", "model.layers.27.mlp.experts.155.down_proj", "model.layers.27.mlp.experts.156.down_proj", "model.layers.27.mlp.experts.157.down_proj", "model.layers.27.mlp.experts.158.down_proj", "model.layers.27.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.1475676223636107e-06, "dbits": 2516582400 } ] }, { "idx": 165, "layers": [ "model.layers.28.self_attn.q_proj" ], "candidates": [ { "dkld": 5.7787488913162675e-06, "dbits": 125829120 } ] }, { "idx": 166, "layers": [ "model.layers.28.self_attn.k_proj", "model.layers.28.self_attn.v_proj" ], "candidates": [ { "dkld": 3.770267358049707e-06, "dbits": 20971520 } ] }, { "idx": 167, "layers": [ "model.layers.28.self_attn.o_proj" ], "candidates": [ { "dkld": 5.4790230933575955e-06, "dbits": 125829120 } ] }, { "idx": 168, "layers": [ "model.layers.28.mlp.shared_experts.gate_proj", "model.layers.28.mlp.shared_experts.up_proj", "model.layers.28.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.5224504750221968e-06, "dbits": 47185920 } ] }, { "idx": 169, "layers": [ "model.layers.28.mlp.experts.0.gate_proj", "model.layers.28.mlp.experts.1.gate_proj", "model.layers.28.mlp.experts.2.gate_proj", "model.layers.28.mlp.experts.3.gate_proj", "model.layers.28.mlp.experts.4.gate_proj", "model.layers.28.mlp.experts.5.gate_proj", "model.layers.28.mlp.experts.6.gate_proj", "model.layers.28.mlp.experts.7.gate_proj", "model.layers.28.mlp.experts.8.gate_proj", "model.layers.28.mlp.experts.9.gate_proj", "model.layers.28.mlp.experts.10.gate_proj", "model.layers.28.mlp.experts.11.gate_proj", "model.layers.28.mlp.experts.12.gate_proj", "model.layers.28.mlp.experts.13.gate_proj", "model.layers.28.mlp.experts.14.gate_proj", "model.layers.28.mlp.experts.15.gate_proj", "model.layers.28.mlp.experts.16.gate_proj", "model.layers.28.mlp.experts.17.gate_proj", "model.layers.28.mlp.experts.18.gate_proj", "model.layers.28.mlp.experts.19.gate_proj", "model.layers.28.mlp.experts.20.gate_proj", "model.layers.28.mlp.experts.21.gate_proj", "model.layers.28.mlp.experts.22.gate_proj", "model.layers.28.mlp.experts.23.gate_proj", "model.layers.28.mlp.experts.24.gate_proj", "model.layers.28.mlp.experts.25.gate_proj", "model.layers.28.mlp.experts.26.gate_proj", "model.layers.28.mlp.experts.27.gate_proj", "model.layers.28.mlp.experts.28.gate_proj", "model.layers.28.mlp.experts.29.gate_proj", "model.layers.28.mlp.experts.30.gate_proj", "model.layers.28.mlp.experts.31.gate_proj", "model.layers.28.mlp.experts.32.gate_proj", "model.layers.28.mlp.experts.33.gate_proj", "model.layers.28.mlp.experts.34.gate_proj", "model.layers.28.mlp.experts.35.gate_proj", "model.layers.28.mlp.experts.36.gate_proj", "model.layers.28.mlp.experts.37.gate_proj", "model.layers.28.mlp.experts.38.gate_proj", "model.layers.28.mlp.experts.39.gate_proj", "model.layers.28.mlp.experts.40.gate_proj", "model.layers.28.mlp.experts.41.gate_proj", "model.layers.28.mlp.experts.42.gate_proj", "model.layers.28.mlp.experts.43.gate_proj", "model.layers.28.mlp.experts.44.gate_proj", "model.layers.28.mlp.experts.45.gate_proj", "model.layers.28.mlp.experts.46.gate_proj", "model.layers.28.mlp.experts.47.gate_proj", "model.layers.28.mlp.experts.48.gate_proj", "model.layers.28.mlp.experts.49.gate_proj", "model.layers.28.mlp.experts.50.gate_proj", "model.layers.28.mlp.experts.51.gate_proj", "model.layers.28.mlp.experts.52.gate_proj", "model.layers.28.mlp.experts.53.gate_proj", "model.layers.28.mlp.experts.54.gate_proj", "model.layers.28.mlp.experts.55.gate_proj", "model.layers.28.mlp.experts.56.gate_proj", "model.layers.28.mlp.experts.57.gate_proj", "model.layers.28.mlp.experts.58.gate_proj", "model.layers.28.mlp.experts.59.gate_proj", "model.layers.28.mlp.experts.60.gate_proj", "model.layers.28.mlp.experts.61.gate_proj", "model.layers.28.mlp.experts.62.gate_proj", "model.layers.28.mlp.experts.63.gate_proj", "model.layers.28.mlp.experts.64.gate_proj", "model.layers.28.mlp.experts.65.gate_proj", "model.layers.28.mlp.experts.66.gate_proj", "model.layers.28.mlp.experts.67.gate_proj", "model.layers.28.mlp.experts.68.gate_proj", "model.layers.28.mlp.experts.69.gate_proj", "model.layers.28.mlp.experts.70.gate_proj", "model.layers.28.mlp.experts.71.gate_proj", "model.layers.28.mlp.experts.72.gate_proj", "model.layers.28.mlp.experts.73.gate_proj", "model.layers.28.mlp.experts.74.gate_proj", "model.layers.28.mlp.experts.75.gate_proj", "model.layers.28.mlp.experts.76.gate_proj", "model.layers.28.mlp.experts.77.gate_proj", "model.layers.28.mlp.experts.78.gate_proj", "model.layers.28.mlp.experts.79.gate_proj", "model.layers.28.mlp.experts.80.gate_proj", "model.layers.28.mlp.experts.81.gate_proj", "model.layers.28.mlp.experts.82.gate_proj", "model.layers.28.mlp.experts.83.gate_proj", "model.layers.28.mlp.experts.84.gate_proj", "model.layers.28.mlp.experts.85.gate_proj", "model.layers.28.mlp.experts.86.gate_proj", "model.layers.28.mlp.experts.87.gate_proj", "model.layers.28.mlp.experts.88.gate_proj", "model.layers.28.mlp.experts.89.gate_proj", "model.layers.28.mlp.experts.90.gate_proj", "model.layers.28.mlp.experts.91.gate_proj", "model.layers.28.mlp.experts.92.gate_proj", "model.layers.28.mlp.experts.93.gate_proj", "model.layers.28.mlp.experts.94.gate_proj", "model.layers.28.mlp.experts.95.gate_proj", "model.layers.28.mlp.experts.96.gate_proj", "model.layers.28.mlp.experts.97.gate_proj", "model.layers.28.mlp.experts.98.gate_proj", "model.layers.28.mlp.experts.99.gate_proj", "model.layers.28.mlp.experts.100.gate_proj", "model.layers.28.mlp.experts.101.gate_proj", "model.layers.28.mlp.experts.102.gate_proj", "model.layers.28.mlp.experts.103.gate_proj", "model.layers.28.mlp.experts.104.gate_proj", "model.layers.28.mlp.experts.105.gate_proj", "model.layers.28.mlp.experts.106.gate_proj", "model.layers.28.mlp.experts.107.gate_proj", "model.layers.28.mlp.experts.108.gate_proj", "model.layers.28.mlp.experts.109.gate_proj", "model.layers.28.mlp.experts.110.gate_proj", "model.layers.28.mlp.experts.111.gate_proj", "model.layers.28.mlp.experts.112.gate_proj", "model.layers.28.mlp.experts.113.gate_proj", "model.layers.28.mlp.experts.114.gate_proj", "model.layers.28.mlp.experts.115.gate_proj", "model.layers.28.mlp.experts.116.gate_proj", "model.layers.28.mlp.experts.117.gate_proj", "model.layers.28.mlp.experts.118.gate_proj", "model.layers.28.mlp.experts.119.gate_proj", "model.layers.28.mlp.experts.120.gate_proj", "model.layers.28.mlp.experts.121.gate_proj", "model.layers.28.mlp.experts.122.gate_proj", "model.layers.28.mlp.experts.123.gate_proj", "model.layers.28.mlp.experts.124.gate_proj", "model.layers.28.mlp.experts.125.gate_proj", "model.layers.28.mlp.experts.126.gate_proj", "model.layers.28.mlp.experts.127.gate_proj", "model.layers.28.mlp.experts.128.gate_proj", "model.layers.28.mlp.experts.129.gate_proj", "model.layers.28.mlp.experts.130.gate_proj", "model.layers.28.mlp.experts.131.gate_proj", "model.layers.28.mlp.experts.132.gate_proj", "model.layers.28.mlp.experts.133.gate_proj", "model.layers.28.mlp.experts.134.gate_proj", "model.layers.28.mlp.experts.135.gate_proj", "model.layers.28.mlp.experts.136.gate_proj", "model.layers.28.mlp.experts.137.gate_proj", "model.layers.28.mlp.experts.138.gate_proj", "model.layers.28.mlp.experts.139.gate_proj", "model.layers.28.mlp.experts.140.gate_proj", "model.layers.28.mlp.experts.141.gate_proj", "model.layers.28.mlp.experts.142.gate_proj", "model.layers.28.mlp.experts.143.gate_proj", "model.layers.28.mlp.experts.144.gate_proj", "model.layers.28.mlp.experts.145.gate_proj", "model.layers.28.mlp.experts.146.gate_proj", "model.layers.28.mlp.experts.147.gate_proj", "model.layers.28.mlp.experts.148.gate_proj", "model.layers.28.mlp.experts.149.gate_proj", "model.layers.28.mlp.experts.150.gate_proj", "model.layers.28.mlp.experts.151.gate_proj", "model.layers.28.mlp.experts.152.gate_proj", "model.layers.28.mlp.experts.153.gate_proj", "model.layers.28.mlp.experts.154.gate_proj", "model.layers.28.mlp.experts.155.gate_proj", "model.layers.28.mlp.experts.156.gate_proj", "model.layers.28.mlp.experts.157.gate_proj", "model.layers.28.mlp.experts.158.gate_proj", "model.layers.28.mlp.experts.159.gate_proj", "model.layers.28.mlp.experts.0.up_proj", "model.layers.28.mlp.experts.1.up_proj", "model.layers.28.mlp.experts.2.up_proj", "model.layers.28.mlp.experts.3.up_proj", "model.layers.28.mlp.experts.4.up_proj", "model.layers.28.mlp.experts.5.up_proj", "model.layers.28.mlp.experts.6.up_proj", "model.layers.28.mlp.experts.7.up_proj", "model.layers.28.mlp.experts.8.up_proj", "model.layers.28.mlp.experts.9.up_proj", "model.layers.28.mlp.experts.10.up_proj", "model.layers.28.mlp.experts.11.up_proj", "model.layers.28.mlp.experts.12.up_proj", "model.layers.28.mlp.experts.13.up_proj", "model.layers.28.mlp.experts.14.up_proj", "model.layers.28.mlp.experts.15.up_proj", "model.layers.28.mlp.experts.16.up_proj", "model.layers.28.mlp.experts.17.up_proj", "model.layers.28.mlp.experts.18.up_proj", "model.layers.28.mlp.experts.19.up_proj", "model.layers.28.mlp.experts.20.up_proj", "model.layers.28.mlp.experts.21.up_proj", "model.layers.28.mlp.experts.22.up_proj", "model.layers.28.mlp.experts.23.up_proj", "model.layers.28.mlp.experts.24.up_proj", "model.layers.28.mlp.experts.25.up_proj", "model.layers.28.mlp.experts.26.up_proj", "model.layers.28.mlp.experts.27.up_proj", "model.layers.28.mlp.experts.28.up_proj", "model.layers.28.mlp.experts.29.up_proj", "model.layers.28.mlp.experts.30.up_proj", "model.layers.28.mlp.experts.31.up_proj", "model.layers.28.mlp.experts.32.up_proj", "model.layers.28.mlp.experts.33.up_proj", "model.layers.28.mlp.experts.34.up_proj", "model.layers.28.mlp.experts.35.up_proj", "model.layers.28.mlp.experts.36.up_proj", "model.layers.28.mlp.experts.37.up_proj", "model.layers.28.mlp.experts.38.up_proj", "model.layers.28.mlp.experts.39.up_proj", "model.layers.28.mlp.experts.40.up_proj", "model.layers.28.mlp.experts.41.up_proj", "model.layers.28.mlp.experts.42.up_proj", "model.layers.28.mlp.experts.43.up_proj", "model.layers.28.mlp.experts.44.up_proj", "model.layers.28.mlp.experts.45.up_proj", "model.layers.28.mlp.experts.46.up_proj", "model.layers.28.mlp.experts.47.up_proj", "model.layers.28.mlp.experts.48.up_proj", "model.layers.28.mlp.experts.49.up_proj", "model.layers.28.mlp.experts.50.up_proj", "model.layers.28.mlp.experts.51.up_proj", "model.layers.28.mlp.experts.52.up_proj", "model.layers.28.mlp.experts.53.up_proj", "model.layers.28.mlp.experts.54.up_proj", "model.layers.28.mlp.experts.55.up_proj", "model.layers.28.mlp.experts.56.up_proj", "model.layers.28.mlp.experts.57.up_proj", "model.layers.28.mlp.experts.58.up_proj", "model.layers.28.mlp.experts.59.up_proj", "model.layers.28.mlp.experts.60.up_proj", "model.layers.28.mlp.experts.61.up_proj", "model.layers.28.mlp.experts.62.up_proj", "model.layers.28.mlp.experts.63.up_proj", "model.layers.28.mlp.experts.64.up_proj", "model.layers.28.mlp.experts.65.up_proj", "model.layers.28.mlp.experts.66.up_proj", "model.layers.28.mlp.experts.67.up_proj", "model.layers.28.mlp.experts.68.up_proj", "model.layers.28.mlp.experts.69.up_proj", "model.layers.28.mlp.experts.70.up_proj", "model.layers.28.mlp.experts.71.up_proj", "model.layers.28.mlp.experts.72.up_proj", "model.layers.28.mlp.experts.73.up_proj", "model.layers.28.mlp.experts.74.up_proj", "model.layers.28.mlp.experts.75.up_proj", "model.layers.28.mlp.experts.76.up_proj", "model.layers.28.mlp.experts.77.up_proj", "model.layers.28.mlp.experts.78.up_proj", "model.layers.28.mlp.experts.79.up_proj", "model.layers.28.mlp.experts.80.up_proj", "model.layers.28.mlp.experts.81.up_proj", "model.layers.28.mlp.experts.82.up_proj", "model.layers.28.mlp.experts.83.up_proj", "model.layers.28.mlp.experts.84.up_proj", "model.layers.28.mlp.experts.85.up_proj", "model.layers.28.mlp.experts.86.up_proj", "model.layers.28.mlp.experts.87.up_proj", "model.layers.28.mlp.experts.88.up_proj", "model.layers.28.mlp.experts.89.up_proj", "model.layers.28.mlp.experts.90.up_proj", "model.layers.28.mlp.experts.91.up_proj", "model.layers.28.mlp.experts.92.up_proj", "model.layers.28.mlp.experts.93.up_proj", "model.layers.28.mlp.experts.94.up_proj", "model.layers.28.mlp.experts.95.up_proj", "model.layers.28.mlp.experts.96.up_proj", "model.layers.28.mlp.experts.97.up_proj", "model.layers.28.mlp.experts.98.up_proj", "model.layers.28.mlp.experts.99.up_proj", "model.layers.28.mlp.experts.100.up_proj", "model.layers.28.mlp.experts.101.up_proj", "model.layers.28.mlp.experts.102.up_proj", "model.layers.28.mlp.experts.103.up_proj", "model.layers.28.mlp.experts.104.up_proj", "model.layers.28.mlp.experts.105.up_proj", "model.layers.28.mlp.experts.106.up_proj", "model.layers.28.mlp.experts.107.up_proj", "model.layers.28.mlp.experts.108.up_proj", "model.layers.28.mlp.experts.109.up_proj", "model.layers.28.mlp.experts.110.up_proj", "model.layers.28.mlp.experts.111.up_proj", "model.layers.28.mlp.experts.112.up_proj", "model.layers.28.mlp.experts.113.up_proj", "model.layers.28.mlp.experts.114.up_proj", "model.layers.28.mlp.experts.115.up_proj", "model.layers.28.mlp.experts.116.up_proj", "model.layers.28.mlp.experts.117.up_proj", "model.layers.28.mlp.experts.118.up_proj", "model.layers.28.mlp.experts.119.up_proj", "model.layers.28.mlp.experts.120.up_proj", "model.layers.28.mlp.experts.121.up_proj", "model.layers.28.mlp.experts.122.up_proj", "model.layers.28.mlp.experts.123.up_proj", "model.layers.28.mlp.experts.124.up_proj", "model.layers.28.mlp.experts.125.up_proj", "model.layers.28.mlp.experts.126.up_proj", "model.layers.28.mlp.experts.127.up_proj", "model.layers.28.mlp.experts.128.up_proj", "model.layers.28.mlp.experts.129.up_proj", "model.layers.28.mlp.experts.130.up_proj", "model.layers.28.mlp.experts.131.up_proj", "model.layers.28.mlp.experts.132.up_proj", "model.layers.28.mlp.experts.133.up_proj", "model.layers.28.mlp.experts.134.up_proj", "model.layers.28.mlp.experts.135.up_proj", "model.layers.28.mlp.experts.136.up_proj", "model.layers.28.mlp.experts.137.up_proj", "model.layers.28.mlp.experts.138.up_proj", "model.layers.28.mlp.experts.139.up_proj", "model.layers.28.mlp.experts.140.up_proj", "model.layers.28.mlp.experts.141.up_proj", "model.layers.28.mlp.experts.142.up_proj", "model.layers.28.mlp.experts.143.up_proj", "model.layers.28.mlp.experts.144.up_proj", "model.layers.28.mlp.experts.145.up_proj", "model.layers.28.mlp.experts.146.up_proj", "model.layers.28.mlp.experts.147.up_proj", "model.layers.28.mlp.experts.148.up_proj", "model.layers.28.mlp.experts.149.up_proj", "model.layers.28.mlp.experts.150.up_proj", "model.layers.28.mlp.experts.151.up_proj", "model.layers.28.mlp.experts.152.up_proj", "model.layers.28.mlp.experts.153.up_proj", "model.layers.28.mlp.experts.154.up_proj", "model.layers.28.mlp.experts.155.up_proj", "model.layers.28.mlp.experts.156.up_proj", "model.layers.28.mlp.experts.157.up_proj", "model.layers.28.mlp.experts.158.up_proj", "model.layers.28.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.5674449969080902e-07, "dbits": 5033164800 } ] }, { "idx": 170, "layers": [ "model.layers.28.mlp.experts.0.down_proj", "model.layers.28.mlp.experts.1.down_proj", "model.layers.28.mlp.experts.2.down_proj", "model.layers.28.mlp.experts.3.down_proj", "model.layers.28.mlp.experts.4.down_proj", "model.layers.28.mlp.experts.5.down_proj", "model.layers.28.mlp.experts.6.down_proj", "model.layers.28.mlp.experts.7.down_proj", "model.layers.28.mlp.experts.8.down_proj", "model.layers.28.mlp.experts.9.down_proj", "model.layers.28.mlp.experts.10.down_proj", "model.layers.28.mlp.experts.11.down_proj", "model.layers.28.mlp.experts.12.down_proj", "model.layers.28.mlp.experts.13.down_proj", "model.layers.28.mlp.experts.14.down_proj", "model.layers.28.mlp.experts.15.down_proj", "model.layers.28.mlp.experts.16.down_proj", "model.layers.28.mlp.experts.17.down_proj", "model.layers.28.mlp.experts.18.down_proj", "model.layers.28.mlp.experts.19.down_proj", "model.layers.28.mlp.experts.20.down_proj", "model.layers.28.mlp.experts.21.down_proj", "model.layers.28.mlp.experts.22.down_proj", "model.layers.28.mlp.experts.23.down_proj", "model.layers.28.mlp.experts.24.down_proj", "model.layers.28.mlp.experts.25.down_proj", "model.layers.28.mlp.experts.26.down_proj", "model.layers.28.mlp.experts.27.down_proj", "model.layers.28.mlp.experts.28.down_proj", "model.layers.28.mlp.experts.29.down_proj", "model.layers.28.mlp.experts.30.down_proj", "model.layers.28.mlp.experts.31.down_proj", "model.layers.28.mlp.experts.32.down_proj", "model.layers.28.mlp.experts.33.down_proj", "model.layers.28.mlp.experts.34.down_proj", "model.layers.28.mlp.experts.35.down_proj", "model.layers.28.mlp.experts.36.down_proj", "model.layers.28.mlp.experts.37.down_proj", "model.layers.28.mlp.experts.38.down_proj", "model.layers.28.mlp.experts.39.down_proj", "model.layers.28.mlp.experts.40.down_proj", "model.layers.28.mlp.experts.41.down_proj", "model.layers.28.mlp.experts.42.down_proj", "model.layers.28.mlp.experts.43.down_proj", "model.layers.28.mlp.experts.44.down_proj", "model.layers.28.mlp.experts.45.down_proj", "model.layers.28.mlp.experts.46.down_proj", "model.layers.28.mlp.experts.47.down_proj", "model.layers.28.mlp.experts.48.down_proj", "model.layers.28.mlp.experts.49.down_proj", "model.layers.28.mlp.experts.50.down_proj", "model.layers.28.mlp.experts.51.down_proj", "model.layers.28.mlp.experts.52.down_proj", "model.layers.28.mlp.experts.53.down_proj", "model.layers.28.mlp.experts.54.down_proj", "model.layers.28.mlp.experts.55.down_proj", "model.layers.28.mlp.experts.56.down_proj", "model.layers.28.mlp.experts.57.down_proj", "model.layers.28.mlp.experts.58.down_proj", "model.layers.28.mlp.experts.59.down_proj", "model.layers.28.mlp.experts.60.down_proj", "model.layers.28.mlp.experts.61.down_proj", "model.layers.28.mlp.experts.62.down_proj", "model.layers.28.mlp.experts.63.down_proj", "model.layers.28.mlp.experts.64.down_proj", "model.layers.28.mlp.experts.65.down_proj", "model.layers.28.mlp.experts.66.down_proj", "model.layers.28.mlp.experts.67.down_proj", "model.layers.28.mlp.experts.68.down_proj", "model.layers.28.mlp.experts.69.down_proj", "model.layers.28.mlp.experts.70.down_proj", "model.layers.28.mlp.experts.71.down_proj", "model.layers.28.mlp.experts.72.down_proj", "model.layers.28.mlp.experts.73.down_proj", "model.layers.28.mlp.experts.74.down_proj", "model.layers.28.mlp.experts.75.down_proj", "model.layers.28.mlp.experts.76.down_proj", "model.layers.28.mlp.experts.77.down_proj", "model.layers.28.mlp.experts.78.down_proj", "model.layers.28.mlp.experts.79.down_proj", "model.layers.28.mlp.experts.80.down_proj", "model.layers.28.mlp.experts.81.down_proj", "model.layers.28.mlp.experts.82.down_proj", "model.layers.28.mlp.experts.83.down_proj", "model.layers.28.mlp.experts.84.down_proj", "model.layers.28.mlp.experts.85.down_proj", "model.layers.28.mlp.experts.86.down_proj", "model.layers.28.mlp.experts.87.down_proj", "model.layers.28.mlp.experts.88.down_proj", "model.layers.28.mlp.experts.89.down_proj", "model.layers.28.mlp.experts.90.down_proj", "model.layers.28.mlp.experts.91.down_proj", "model.layers.28.mlp.experts.92.down_proj", "model.layers.28.mlp.experts.93.down_proj", "model.layers.28.mlp.experts.94.down_proj", "model.layers.28.mlp.experts.95.down_proj", "model.layers.28.mlp.experts.96.down_proj", "model.layers.28.mlp.experts.97.down_proj", "model.layers.28.mlp.experts.98.down_proj", "model.layers.28.mlp.experts.99.down_proj", "model.layers.28.mlp.experts.100.down_proj", "model.layers.28.mlp.experts.101.down_proj", "model.layers.28.mlp.experts.102.down_proj", "model.layers.28.mlp.experts.103.down_proj", "model.layers.28.mlp.experts.104.down_proj", "model.layers.28.mlp.experts.105.down_proj", "model.layers.28.mlp.experts.106.down_proj", "model.layers.28.mlp.experts.107.down_proj", "model.layers.28.mlp.experts.108.down_proj", "model.layers.28.mlp.experts.109.down_proj", "model.layers.28.mlp.experts.110.down_proj", "model.layers.28.mlp.experts.111.down_proj", "model.layers.28.mlp.experts.112.down_proj", "model.layers.28.mlp.experts.113.down_proj", "model.layers.28.mlp.experts.114.down_proj", "model.layers.28.mlp.experts.115.down_proj", "model.layers.28.mlp.experts.116.down_proj", "model.layers.28.mlp.experts.117.down_proj", "model.layers.28.mlp.experts.118.down_proj", "model.layers.28.mlp.experts.119.down_proj", "model.layers.28.mlp.experts.120.down_proj", "model.layers.28.mlp.experts.121.down_proj", "model.layers.28.mlp.experts.122.down_proj", "model.layers.28.mlp.experts.123.down_proj", "model.layers.28.mlp.experts.124.down_proj", "model.layers.28.mlp.experts.125.down_proj", "model.layers.28.mlp.experts.126.down_proj", "model.layers.28.mlp.experts.127.down_proj", "model.layers.28.mlp.experts.128.down_proj", "model.layers.28.mlp.experts.129.down_proj", "model.layers.28.mlp.experts.130.down_proj", "model.layers.28.mlp.experts.131.down_proj", "model.layers.28.mlp.experts.132.down_proj", "model.layers.28.mlp.experts.133.down_proj", "model.layers.28.mlp.experts.134.down_proj", "model.layers.28.mlp.experts.135.down_proj", "model.layers.28.mlp.experts.136.down_proj", "model.layers.28.mlp.experts.137.down_proj", "model.layers.28.mlp.experts.138.down_proj", "model.layers.28.mlp.experts.139.down_proj", "model.layers.28.mlp.experts.140.down_proj", "model.layers.28.mlp.experts.141.down_proj", "model.layers.28.mlp.experts.142.down_proj", "model.layers.28.mlp.experts.143.down_proj", "model.layers.28.mlp.experts.144.down_proj", "model.layers.28.mlp.experts.145.down_proj", "model.layers.28.mlp.experts.146.down_proj", "model.layers.28.mlp.experts.147.down_proj", "model.layers.28.mlp.experts.148.down_proj", "model.layers.28.mlp.experts.149.down_proj", "model.layers.28.mlp.experts.150.down_proj", "model.layers.28.mlp.experts.151.down_proj", "model.layers.28.mlp.experts.152.down_proj", "model.layers.28.mlp.experts.153.down_proj", "model.layers.28.mlp.experts.154.down_proj", "model.layers.28.mlp.experts.155.down_proj", "model.layers.28.mlp.experts.156.down_proj", "model.layers.28.mlp.experts.157.down_proj", "model.layers.28.mlp.experts.158.down_proj", "model.layers.28.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.14792380575095e-06, "dbits": 2516582400 } ] }, { "idx": 171, "layers": [ "model.layers.29.self_attn.q_proj" ], "candidates": [ { "dkld": 6.610876880586364e-07, "dbits": 125829120 } ] }, { "idx": 172, "layers": [ "model.layers.29.self_attn.k_proj", "model.layers.29.self_attn.v_proj" ], "candidates": [ { "dkld": -6.277210195548854e-06, "dbits": 20971520 } ] }, { "idx": 173, "layers": [ "model.layers.29.self_attn.o_proj" ], "candidates": [ { "dkld": -2.6282272301614068e-06, "dbits": 125829120 } ] }, { "idx": 174, "layers": [ "model.layers.29.mlp.shared_experts.gate_proj", "model.layers.29.mlp.shared_experts.up_proj", "model.layers.29.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -3.314533387310803e-06, "dbits": 47185920 } ] }, { "idx": 175, "layers": [ "model.layers.29.mlp.experts.0.gate_proj", "model.layers.29.mlp.experts.1.gate_proj", "model.layers.29.mlp.experts.2.gate_proj", "model.layers.29.mlp.experts.3.gate_proj", "model.layers.29.mlp.experts.4.gate_proj", "model.layers.29.mlp.experts.5.gate_proj", "model.layers.29.mlp.experts.6.gate_proj", "model.layers.29.mlp.experts.7.gate_proj", "model.layers.29.mlp.experts.8.gate_proj", "model.layers.29.mlp.experts.9.gate_proj", "model.layers.29.mlp.experts.10.gate_proj", "model.layers.29.mlp.experts.11.gate_proj", "model.layers.29.mlp.experts.12.gate_proj", "model.layers.29.mlp.experts.13.gate_proj", "model.layers.29.mlp.experts.14.gate_proj", "model.layers.29.mlp.experts.15.gate_proj", "model.layers.29.mlp.experts.16.gate_proj", "model.layers.29.mlp.experts.17.gate_proj", "model.layers.29.mlp.experts.18.gate_proj", "model.layers.29.mlp.experts.19.gate_proj", "model.layers.29.mlp.experts.20.gate_proj", "model.layers.29.mlp.experts.21.gate_proj", "model.layers.29.mlp.experts.22.gate_proj", "model.layers.29.mlp.experts.23.gate_proj", "model.layers.29.mlp.experts.24.gate_proj", "model.layers.29.mlp.experts.25.gate_proj", "model.layers.29.mlp.experts.26.gate_proj", "model.layers.29.mlp.experts.27.gate_proj", "model.layers.29.mlp.experts.28.gate_proj", "model.layers.29.mlp.experts.29.gate_proj", "model.layers.29.mlp.experts.30.gate_proj", "model.layers.29.mlp.experts.31.gate_proj", "model.layers.29.mlp.experts.32.gate_proj", "model.layers.29.mlp.experts.33.gate_proj", "model.layers.29.mlp.experts.34.gate_proj", "model.layers.29.mlp.experts.35.gate_proj", "model.layers.29.mlp.experts.36.gate_proj", "model.layers.29.mlp.experts.37.gate_proj", "model.layers.29.mlp.experts.38.gate_proj", "model.layers.29.mlp.experts.39.gate_proj", "model.layers.29.mlp.experts.40.gate_proj", "model.layers.29.mlp.experts.41.gate_proj", "model.layers.29.mlp.experts.42.gate_proj", "model.layers.29.mlp.experts.43.gate_proj", "model.layers.29.mlp.experts.44.gate_proj", "model.layers.29.mlp.experts.45.gate_proj", "model.layers.29.mlp.experts.46.gate_proj", "model.layers.29.mlp.experts.47.gate_proj", "model.layers.29.mlp.experts.48.gate_proj", "model.layers.29.mlp.experts.49.gate_proj", "model.layers.29.mlp.experts.50.gate_proj", "model.layers.29.mlp.experts.51.gate_proj", "model.layers.29.mlp.experts.52.gate_proj", "model.layers.29.mlp.experts.53.gate_proj", "model.layers.29.mlp.experts.54.gate_proj", "model.layers.29.mlp.experts.55.gate_proj", "model.layers.29.mlp.experts.56.gate_proj", "model.layers.29.mlp.experts.57.gate_proj", "model.layers.29.mlp.experts.58.gate_proj", "model.layers.29.mlp.experts.59.gate_proj", "model.layers.29.mlp.experts.60.gate_proj", "model.layers.29.mlp.experts.61.gate_proj", "model.layers.29.mlp.experts.62.gate_proj", "model.layers.29.mlp.experts.63.gate_proj", "model.layers.29.mlp.experts.64.gate_proj", "model.layers.29.mlp.experts.65.gate_proj", "model.layers.29.mlp.experts.66.gate_proj", "model.layers.29.mlp.experts.67.gate_proj", "model.layers.29.mlp.experts.68.gate_proj", "model.layers.29.mlp.experts.69.gate_proj", "model.layers.29.mlp.experts.70.gate_proj", "model.layers.29.mlp.experts.71.gate_proj", "model.layers.29.mlp.experts.72.gate_proj", "model.layers.29.mlp.experts.73.gate_proj", "model.layers.29.mlp.experts.74.gate_proj", "model.layers.29.mlp.experts.75.gate_proj", "model.layers.29.mlp.experts.76.gate_proj", "model.layers.29.mlp.experts.77.gate_proj", "model.layers.29.mlp.experts.78.gate_proj", "model.layers.29.mlp.experts.79.gate_proj", "model.layers.29.mlp.experts.80.gate_proj", "model.layers.29.mlp.experts.81.gate_proj", "model.layers.29.mlp.experts.82.gate_proj", "model.layers.29.mlp.experts.83.gate_proj", "model.layers.29.mlp.experts.84.gate_proj", "model.layers.29.mlp.experts.85.gate_proj", "model.layers.29.mlp.experts.86.gate_proj", "model.layers.29.mlp.experts.87.gate_proj", "model.layers.29.mlp.experts.88.gate_proj", "model.layers.29.mlp.experts.89.gate_proj", "model.layers.29.mlp.experts.90.gate_proj", "model.layers.29.mlp.experts.91.gate_proj", "model.layers.29.mlp.experts.92.gate_proj", "model.layers.29.mlp.experts.93.gate_proj", "model.layers.29.mlp.experts.94.gate_proj", "model.layers.29.mlp.experts.95.gate_proj", "model.layers.29.mlp.experts.96.gate_proj", "model.layers.29.mlp.experts.97.gate_proj", "model.layers.29.mlp.experts.98.gate_proj", "model.layers.29.mlp.experts.99.gate_proj", "model.layers.29.mlp.experts.100.gate_proj", "model.layers.29.mlp.experts.101.gate_proj", "model.layers.29.mlp.experts.102.gate_proj", "model.layers.29.mlp.experts.103.gate_proj", "model.layers.29.mlp.experts.104.gate_proj", "model.layers.29.mlp.experts.105.gate_proj", "model.layers.29.mlp.experts.106.gate_proj", "model.layers.29.mlp.experts.107.gate_proj", "model.layers.29.mlp.experts.108.gate_proj", "model.layers.29.mlp.experts.109.gate_proj", "model.layers.29.mlp.experts.110.gate_proj", "model.layers.29.mlp.experts.111.gate_proj", "model.layers.29.mlp.experts.112.gate_proj", "model.layers.29.mlp.experts.113.gate_proj", "model.layers.29.mlp.experts.114.gate_proj", "model.layers.29.mlp.experts.115.gate_proj", "model.layers.29.mlp.experts.116.gate_proj", "model.layers.29.mlp.experts.117.gate_proj", "model.layers.29.mlp.experts.118.gate_proj", "model.layers.29.mlp.experts.119.gate_proj", "model.layers.29.mlp.experts.120.gate_proj", "model.layers.29.mlp.experts.121.gate_proj", "model.layers.29.mlp.experts.122.gate_proj", "model.layers.29.mlp.experts.123.gate_proj", "model.layers.29.mlp.experts.124.gate_proj", "model.layers.29.mlp.experts.125.gate_proj", "model.layers.29.mlp.experts.126.gate_proj", "model.layers.29.mlp.experts.127.gate_proj", "model.layers.29.mlp.experts.128.gate_proj", "model.layers.29.mlp.experts.129.gate_proj", "model.layers.29.mlp.experts.130.gate_proj", "model.layers.29.mlp.experts.131.gate_proj", "model.layers.29.mlp.experts.132.gate_proj", "model.layers.29.mlp.experts.133.gate_proj", "model.layers.29.mlp.experts.134.gate_proj", "model.layers.29.mlp.experts.135.gate_proj", "model.layers.29.mlp.experts.136.gate_proj", "model.layers.29.mlp.experts.137.gate_proj", "model.layers.29.mlp.experts.138.gate_proj", "model.layers.29.mlp.experts.139.gate_proj", "model.layers.29.mlp.experts.140.gate_proj", "model.layers.29.mlp.experts.141.gate_proj", "model.layers.29.mlp.experts.142.gate_proj", "model.layers.29.mlp.experts.143.gate_proj", "model.layers.29.mlp.experts.144.gate_proj", "model.layers.29.mlp.experts.145.gate_proj", "model.layers.29.mlp.experts.146.gate_proj", "model.layers.29.mlp.experts.147.gate_proj", "model.layers.29.mlp.experts.148.gate_proj", "model.layers.29.mlp.experts.149.gate_proj", "model.layers.29.mlp.experts.150.gate_proj", "model.layers.29.mlp.experts.151.gate_proj", "model.layers.29.mlp.experts.152.gate_proj", "model.layers.29.mlp.experts.153.gate_proj", "model.layers.29.mlp.experts.154.gate_proj", "model.layers.29.mlp.experts.155.gate_proj", "model.layers.29.mlp.experts.156.gate_proj", "model.layers.29.mlp.experts.157.gate_proj", "model.layers.29.mlp.experts.158.gate_proj", "model.layers.29.mlp.experts.159.gate_proj", "model.layers.29.mlp.experts.0.up_proj", "model.layers.29.mlp.experts.1.up_proj", "model.layers.29.mlp.experts.2.up_proj", "model.layers.29.mlp.experts.3.up_proj", "model.layers.29.mlp.experts.4.up_proj", "model.layers.29.mlp.experts.5.up_proj", "model.layers.29.mlp.experts.6.up_proj", "model.layers.29.mlp.experts.7.up_proj", "model.layers.29.mlp.experts.8.up_proj", "model.layers.29.mlp.experts.9.up_proj", "model.layers.29.mlp.experts.10.up_proj", "model.layers.29.mlp.experts.11.up_proj", "model.layers.29.mlp.experts.12.up_proj", "model.layers.29.mlp.experts.13.up_proj", "model.layers.29.mlp.experts.14.up_proj", "model.layers.29.mlp.experts.15.up_proj", "model.layers.29.mlp.experts.16.up_proj", "model.layers.29.mlp.experts.17.up_proj", "model.layers.29.mlp.experts.18.up_proj", "model.layers.29.mlp.experts.19.up_proj", "model.layers.29.mlp.experts.20.up_proj", "model.layers.29.mlp.experts.21.up_proj", "model.layers.29.mlp.experts.22.up_proj", "model.layers.29.mlp.experts.23.up_proj", "model.layers.29.mlp.experts.24.up_proj", "model.layers.29.mlp.experts.25.up_proj", "model.layers.29.mlp.experts.26.up_proj", "model.layers.29.mlp.experts.27.up_proj", "model.layers.29.mlp.experts.28.up_proj", "model.layers.29.mlp.experts.29.up_proj", "model.layers.29.mlp.experts.30.up_proj", "model.layers.29.mlp.experts.31.up_proj", "model.layers.29.mlp.experts.32.up_proj", "model.layers.29.mlp.experts.33.up_proj", "model.layers.29.mlp.experts.34.up_proj", "model.layers.29.mlp.experts.35.up_proj", "model.layers.29.mlp.experts.36.up_proj", "model.layers.29.mlp.experts.37.up_proj", "model.layers.29.mlp.experts.38.up_proj", "model.layers.29.mlp.experts.39.up_proj", "model.layers.29.mlp.experts.40.up_proj", "model.layers.29.mlp.experts.41.up_proj", "model.layers.29.mlp.experts.42.up_proj", "model.layers.29.mlp.experts.43.up_proj", "model.layers.29.mlp.experts.44.up_proj", "model.layers.29.mlp.experts.45.up_proj", "model.layers.29.mlp.experts.46.up_proj", "model.layers.29.mlp.experts.47.up_proj", "model.layers.29.mlp.experts.48.up_proj", "model.layers.29.mlp.experts.49.up_proj", "model.layers.29.mlp.experts.50.up_proj", "model.layers.29.mlp.experts.51.up_proj", "model.layers.29.mlp.experts.52.up_proj", "model.layers.29.mlp.experts.53.up_proj", "model.layers.29.mlp.experts.54.up_proj", "model.layers.29.mlp.experts.55.up_proj", "model.layers.29.mlp.experts.56.up_proj", "model.layers.29.mlp.experts.57.up_proj", "model.layers.29.mlp.experts.58.up_proj", "model.layers.29.mlp.experts.59.up_proj", "model.layers.29.mlp.experts.60.up_proj", "model.layers.29.mlp.experts.61.up_proj", "model.layers.29.mlp.experts.62.up_proj", "model.layers.29.mlp.experts.63.up_proj", "model.layers.29.mlp.experts.64.up_proj", "model.layers.29.mlp.experts.65.up_proj", "model.layers.29.mlp.experts.66.up_proj", "model.layers.29.mlp.experts.67.up_proj", "model.layers.29.mlp.experts.68.up_proj", "model.layers.29.mlp.experts.69.up_proj", "model.layers.29.mlp.experts.70.up_proj", "model.layers.29.mlp.experts.71.up_proj", "model.layers.29.mlp.experts.72.up_proj", "model.layers.29.mlp.experts.73.up_proj", "model.layers.29.mlp.experts.74.up_proj", "model.layers.29.mlp.experts.75.up_proj", "model.layers.29.mlp.experts.76.up_proj", "model.layers.29.mlp.experts.77.up_proj", "model.layers.29.mlp.experts.78.up_proj", "model.layers.29.mlp.experts.79.up_proj", "model.layers.29.mlp.experts.80.up_proj", "model.layers.29.mlp.experts.81.up_proj", "model.layers.29.mlp.experts.82.up_proj", "model.layers.29.mlp.experts.83.up_proj", "model.layers.29.mlp.experts.84.up_proj", "model.layers.29.mlp.experts.85.up_proj", "model.layers.29.mlp.experts.86.up_proj", "model.layers.29.mlp.experts.87.up_proj", "model.layers.29.mlp.experts.88.up_proj", "model.layers.29.mlp.experts.89.up_proj", "model.layers.29.mlp.experts.90.up_proj", "model.layers.29.mlp.experts.91.up_proj", "model.layers.29.mlp.experts.92.up_proj", "model.layers.29.mlp.experts.93.up_proj", "model.layers.29.mlp.experts.94.up_proj", "model.layers.29.mlp.experts.95.up_proj", "model.layers.29.mlp.experts.96.up_proj", "model.layers.29.mlp.experts.97.up_proj", "model.layers.29.mlp.experts.98.up_proj", "model.layers.29.mlp.experts.99.up_proj", "model.layers.29.mlp.experts.100.up_proj", "model.layers.29.mlp.experts.101.up_proj", "model.layers.29.mlp.experts.102.up_proj", "model.layers.29.mlp.experts.103.up_proj", "model.layers.29.mlp.experts.104.up_proj", "model.layers.29.mlp.experts.105.up_proj", "model.layers.29.mlp.experts.106.up_proj", "model.layers.29.mlp.experts.107.up_proj", "model.layers.29.mlp.experts.108.up_proj", "model.layers.29.mlp.experts.109.up_proj", "model.layers.29.mlp.experts.110.up_proj", "model.layers.29.mlp.experts.111.up_proj", "model.layers.29.mlp.experts.112.up_proj", "model.layers.29.mlp.experts.113.up_proj", "model.layers.29.mlp.experts.114.up_proj", "model.layers.29.mlp.experts.115.up_proj", "model.layers.29.mlp.experts.116.up_proj", "model.layers.29.mlp.experts.117.up_proj", "model.layers.29.mlp.experts.118.up_proj", "model.layers.29.mlp.experts.119.up_proj", "model.layers.29.mlp.experts.120.up_proj", "model.layers.29.mlp.experts.121.up_proj", "model.layers.29.mlp.experts.122.up_proj", "model.layers.29.mlp.experts.123.up_proj", "model.layers.29.mlp.experts.124.up_proj", "model.layers.29.mlp.experts.125.up_proj", "model.layers.29.mlp.experts.126.up_proj", "model.layers.29.mlp.experts.127.up_proj", "model.layers.29.mlp.experts.128.up_proj", "model.layers.29.mlp.experts.129.up_proj", "model.layers.29.mlp.experts.130.up_proj", "model.layers.29.mlp.experts.131.up_proj", "model.layers.29.mlp.experts.132.up_proj", "model.layers.29.mlp.experts.133.up_proj", "model.layers.29.mlp.experts.134.up_proj", "model.layers.29.mlp.experts.135.up_proj", "model.layers.29.mlp.experts.136.up_proj", "model.layers.29.mlp.experts.137.up_proj", "model.layers.29.mlp.experts.138.up_proj", "model.layers.29.mlp.experts.139.up_proj", "model.layers.29.mlp.experts.140.up_proj", "model.layers.29.mlp.experts.141.up_proj", "model.layers.29.mlp.experts.142.up_proj", "model.layers.29.mlp.experts.143.up_proj", "model.layers.29.mlp.experts.144.up_proj", "model.layers.29.mlp.experts.145.up_proj", "model.layers.29.mlp.experts.146.up_proj", "model.layers.29.mlp.experts.147.up_proj", "model.layers.29.mlp.experts.148.up_proj", "model.layers.29.mlp.experts.149.up_proj", "model.layers.29.mlp.experts.150.up_proj", "model.layers.29.mlp.experts.151.up_proj", "model.layers.29.mlp.experts.152.up_proj", "model.layers.29.mlp.experts.153.up_proj", "model.layers.29.mlp.experts.154.up_proj", "model.layers.29.mlp.experts.155.up_proj", "model.layers.29.mlp.experts.156.up_proj", "model.layers.29.mlp.experts.157.up_proj", "model.layers.29.mlp.experts.158.up_proj", "model.layers.29.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.0128361433766115e-08, "dbits": 5033164800 } ] }, { "idx": 176, "layers": [ "model.layers.29.mlp.experts.0.down_proj", "model.layers.29.mlp.experts.1.down_proj", "model.layers.29.mlp.experts.2.down_proj", "model.layers.29.mlp.experts.3.down_proj", "model.layers.29.mlp.experts.4.down_proj", "model.layers.29.mlp.experts.5.down_proj", "model.layers.29.mlp.experts.6.down_proj", "model.layers.29.mlp.experts.7.down_proj", "model.layers.29.mlp.experts.8.down_proj", "model.layers.29.mlp.experts.9.down_proj", "model.layers.29.mlp.experts.10.down_proj", "model.layers.29.mlp.experts.11.down_proj", "model.layers.29.mlp.experts.12.down_proj", "model.layers.29.mlp.experts.13.down_proj", "model.layers.29.mlp.experts.14.down_proj", "model.layers.29.mlp.experts.15.down_proj", "model.layers.29.mlp.experts.16.down_proj", "model.layers.29.mlp.experts.17.down_proj", "model.layers.29.mlp.experts.18.down_proj", "model.layers.29.mlp.experts.19.down_proj", "model.layers.29.mlp.experts.20.down_proj", "model.layers.29.mlp.experts.21.down_proj", "model.layers.29.mlp.experts.22.down_proj", "model.layers.29.mlp.experts.23.down_proj", "model.layers.29.mlp.experts.24.down_proj", "model.layers.29.mlp.experts.25.down_proj", "model.layers.29.mlp.experts.26.down_proj", "model.layers.29.mlp.experts.27.down_proj", "model.layers.29.mlp.experts.28.down_proj", "model.layers.29.mlp.experts.29.down_proj", "model.layers.29.mlp.experts.30.down_proj", "model.layers.29.mlp.experts.31.down_proj", "model.layers.29.mlp.experts.32.down_proj", "model.layers.29.mlp.experts.33.down_proj", "model.layers.29.mlp.experts.34.down_proj", "model.layers.29.mlp.experts.35.down_proj", "model.layers.29.mlp.experts.36.down_proj", "model.layers.29.mlp.experts.37.down_proj", "model.layers.29.mlp.experts.38.down_proj", "model.layers.29.mlp.experts.39.down_proj", "model.layers.29.mlp.experts.40.down_proj", "model.layers.29.mlp.experts.41.down_proj", "model.layers.29.mlp.experts.42.down_proj", "model.layers.29.mlp.experts.43.down_proj", "model.layers.29.mlp.experts.44.down_proj", "model.layers.29.mlp.experts.45.down_proj", "model.layers.29.mlp.experts.46.down_proj", "model.layers.29.mlp.experts.47.down_proj", "model.layers.29.mlp.experts.48.down_proj", "model.layers.29.mlp.experts.49.down_proj", "model.layers.29.mlp.experts.50.down_proj", "model.layers.29.mlp.experts.51.down_proj", "model.layers.29.mlp.experts.52.down_proj", "model.layers.29.mlp.experts.53.down_proj", "model.layers.29.mlp.experts.54.down_proj", "model.layers.29.mlp.experts.55.down_proj", "model.layers.29.mlp.experts.56.down_proj", "model.layers.29.mlp.experts.57.down_proj", "model.layers.29.mlp.experts.58.down_proj", "model.layers.29.mlp.experts.59.down_proj", "model.layers.29.mlp.experts.60.down_proj", "model.layers.29.mlp.experts.61.down_proj", "model.layers.29.mlp.experts.62.down_proj", "model.layers.29.mlp.experts.63.down_proj", "model.layers.29.mlp.experts.64.down_proj", "model.layers.29.mlp.experts.65.down_proj", "model.layers.29.mlp.experts.66.down_proj", "model.layers.29.mlp.experts.67.down_proj", "model.layers.29.mlp.experts.68.down_proj", "model.layers.29.mlp.experts.69.down_proj", "model.layers.29.mlp.experts.70.down_proj", "model.layers.29.mlp.experts.71.down_proj", "model.layers.29.mlp.experts.72.down_proj", "model.layers.29.mlp.experts.73.down_proj", "model.layers.29.mlp.experts.74.down_proj", "model.layers.29.mlp.experts.75.down_proj", "model.layers.29.mlp.experts.76.down_proj", "model.layers.29.mlp.experts.77.down_proj", "model.layers.29.mlp.experts.78.down_proj", "model.layers.29.mlp.experts.79.down_proj", "model.layers.29.mlp.experts.80.down_proj", "model.layers.29.mlp.experts.81.down_proj", "model.layers.29.mlp.experts.82.down_proj", "model.layers.29.mlp.experts.83.down_proj", "model.layers.29.mlp.experts.84.down_proj", "model.layers.29.mlp.experts.85.down_proj", "model.layers.29.mlp.experts.86.down_proj", "model.layers.29.mlp.experts.87.down_proj", "model.layers.29.mlp.experts.88.down_proj", "model.layers.29.mlp.experts.89.down_proj", "model.layers.29.mlp.experts.90.down_proj", "model.layers.29.mlp.experts.91.down_proj", "model.layers.29.mlp.experts.92.down_proj", "model.layers.29.mlp.experts.93.down_proj", "model.layers.29.mlp.experts.94.down_proj", "model.layers.29.mlp.experts.95.down_proj", "model.layers.29.mlp.experts.96.down_proj", "model.layers.29.mlp.experts.97.down_proj", "model.layers.29.mlp.experts.98.down_proj", "model.layers.29.mlp.experts.99.down_proj", "model.layers.29.mlp.experts.100.down_proj", "model.layers.29.mlp.experts.101.down_proj", "model.layers.29.mlp.experts.102.down_proj", "model.layers.29.mlp.experts.103.down_proj", "model.layers.29.mlp.experts.104.down_proj", "model.layers.29.mlp.experts.105.down_proj", "model.layers.29.mlp.experts.106.down_proj", "model.layers.29.mlp.experts.107.down_proj", "model.layers.29.mlp.experts.108.down_proj", "model.layers.29.mlp.experts.109.down_proj", "model.layers.29.mlp.experts.110.down_proj", "model.layers.29.mlp.experts.111.down_proj", "model.layers.29.mlp.experts.112.down_proj", "model.layers.29.mlp.experts.113.down_proj", "model.layers.29.mlp.experts.114.down_proj", "model.layers.29.mlp.experts.115.down_proj", "model.layers.29.mlp.experts.116.down_proj", "model.layers.29.mlp.experts.117.down_proj", "model.layers.29.mlp.experts.118.down_proj", "model.layers.29.mlp.experts.119.down_proj", "model.layers.29.mlp.experts.120.down_proj", "model.layers.29.mlp.experts.121.down_proj", "model.layers.29.mlp.experts.122.down_proj", "model.layers.29.mlp.experts.123.down_proj", "model.layers.29.mlp.experts.124.down_proj", "model.layers.29.mlp.experts.125.down_proj", "model.layers.29.mlp.experts.126.down_proj", "model.layers.29.mlp.experts.127.down_proj", "model.layers.29.mlp.experts.128.down_proj", "model.layers.29.mlp.experts.129.down_proj", "model.layers.29.mlp.experts.130.down_proj", "model.layers.29.mlp.experts.131.down_proj", "model.layers.29.mlp.experts.132.down_proj", "model.layers.29.mlp.experts.133.down_proj", "model.layers.29.mlp.experts.134.down_proj", "model.layers.29.mlp.experts.135.down_proj", "model.layers.29.mlp.experts.136.down_proj", "model.layers.29.mlp.experts.137.down_proj", "model.layers.29.mlp.experts.138.down_proj", "model.layers.29.mlp.experts.139.down_proj", "model.layers.29.mlp.experts.140.down_proj", "model.layers.29.mlp.experts.141.down_proj", "model.layers.29.mlp.experts.142.down_proj", "model.layers.29.mlp.experts.143.down_proj", "model.layers.29.mlp.experts.144.down_proj", "model.layers.29.mlp.experts.145.down_proj", "model.layers.29.mlp.experts.146.down_proj", "model.layers.29.mlp.experts.147.down_proj", "model.layers.29.mlp.experts.148.down_proj", "model.layers.29.mlp.experts.149.down_proj", "model.layers.29.mlp.experts.150.down_proj", "model.layers.29.mlp.experts.151.down_proj", "model.layers.29.mlp.experts.152.down_proj", "model.layers.29.mlp.experts.153.down_proj", "model.layers.29.mlp.experts.154.down_proj", "model.layers.29.mlp.experts.155.down_proj", "model.layers.29.mlp.experts.156.down_proj", "model.layers.29.mlp.experts.157.down_proj", "model.layers.29.mlp.experts.158.down_proj", "model.layers.29.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.6777852326633714e-07, "dbits": 2516582400 } ] }, { "idx": 177, "layers": [ "model.layers.30.self_attn.q_proj" ], "candidates": [ { "dkld": -5.328934639692328e-06, "dbits": 125829120 } ] }, { "idx": 178, "layers": [ "model.layers.30.self_attn.k_proj", "model.layers.30.self_attn.v_proj" ], "candidates": [ { "dkld": -2.763466909527562e-07, "dbits": 20971520 } ] }, { "idx": 179, "layers": [ "model.layers.30.self_attn.o_proj" ], "candidates": [ { "dkld": -3.0215451261028647e-06, "dbits": 125829120 } ] }, { "idx": 180, "layers": [ "model.layers.30.mlp.shared_experts.gate_proj", "model.layers.30.mlp.shared_experts.up_proj", "model.layers.30.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.1100855772383511e-05, "dbits": 47185920 } ] }, { "idx": 181, "layers": [ "model.layers.30.mlp.experts.0.gate_proj", "model.layers.30.mlp.experts.1.gate_proj", "model.layers.30.mlp.experts.2.gate_proj", "model.layers.30.mlp.experts.3.gate_proj", "model.layers.30.mlp.experts.4.gate_proj", "model.layers.30.mlp.experts.5.gate_proj", "model.layers.30.mlp.experts.6.gate_proj", "model.layers.30.mlp.experts.7.gate_proj", "model.layers.30.mlp.experts.8.gate_proj", "model.layers.30.mlp.experts.9.gate_proj", "model.layers.30.mlp.experts.10.gate_proj", "model.layers.30.mlp.experts.11.gate_proj", "model.layers.30.mlp.experts.12.gate_proj", "model.layers.30.mlp.experts.13.gate_proj", "model.layers.30.mlp.experts.14.gate_proj", "model.layers.30.mlp.experts.15.gate_proj", "model.layers.30.mlp.experts.16.gate_proj", "model.layers.30.mlp.experts.17.gate_proj", "model.layers.30.mlp.experts.18.gate_proj", "model.layers.30.mlp.experts.19.gate_proj", "model.layers.30.mlp.experts.20.gate_proj", "model.layers.30.mlp.experts.21.gate_proj", "model.layers.30.mlp.experts.22.gate_proj", "model.layers.30.mlp.experts.23.gate_proj", "model.layers.30.mlp.experts.24.gate_proj", "model.layers.30.mlp.experts.25.gate_proj", "model.layers.30.mlp.experts.26.gate_proj", "model.layers.30.mlp.experts.27.gate_proj", "model.layers.30.mlp.experts.28.gate_proj", "model.layers.30.mlp.experts.29.gate_proj", "model.layers.30.mlp.experts.30.gate_proj", "model.layers.30.mlp.experts.31.gate_proj", "model.layers.30.mlp.experts.32.gate_proj", "model.layers.30.mlp.experts.33.gate_proj", "model.layers.30.mlp.experts.34.gate_proj", "model.layers.30.mlp.experts.35.gate_proj", "model.layers.30.mlp.experts.36.gate_proj", "model.layers.30.mlp.experts.37.gate_proj", "model.layers.30.mlp.experts.38.gate_proj", "model.layers.30.mlp.experts.39.gate_proj", "model.layers.30.mlp.experts.40.gate_proj", "model.layers.30.mlp.experts.41.gate_proj", "model.layers.30.mlp.experts.42.gate_proj", "model.layers.30.mlp.experts.43.gate_proj", "model.layers.30.mlp.experts.44.gate_proj", "model.layers.30.mlp.experts.45.gate_proj", "model.layers.30.mlp.experts.46.gate_proj", "model.layers.30.mlp.experts.47.gate_proj", "model.layers.30.mlp.experts.48.gate_proj", "model.layers.30.mlp.experts.49.gate_proj", "model.layers.30.mlp.experts.50.gate_proj", "model.layers.30.mlp.experts.51.gate_proj", "model.layers.30.mlp.experts.52.gate_proj", "model.layers.30.mlp.experts.53.gate_proj", "model.layers.30.mlp.experts.54.gate_proj", "model.layers.30.mlp.experts.55.gate_proj", "model.layers.30.mlp.experts.56.gate_proj", "model.layers.30.mlp.experts.57.gate_proj", "model.layers.30.mlp.experts.58.gate_proj", "model.layers.30.mlp.experts.59.gate_proj", "model.layers.30.mlp.experts.60.gate_proj", "model.layers.30.mlp.experts.61.gate_proj", "model.layers.30.mlp.experts.62.gate_proj", "model.layers.30.mlp.experts.63.gate_proj", "model.layers.30.mlp.experts.64.gate_proj", "model.layers.30.mlp.experts.65.gate_proj", "model.layers.30.mlp.experts.66.gate_proj", "model.layers.30.mlp.experts.67.gate_proj", "model.layers.30.mlp.experts.68.gate_proj", "model.layers.30.mlp.experts.69.gate_proj", "model.layers.30.mlp.experts.70.gate_proj", "model.layers.30.mlp.experts.71.gate_proj", "model.layers.30.mlp.experts.72.gate_proj", "model.layers.30.mlp.experts.73.gate_proj", "model.layers.30.mlp.experts.74.gate_proj", "model.layers.30.mlp.experts.75.gate_proj", "model.layers.30.mlp.experts.76.gate_proj", "model.layers.30.mlp.experts.77.gate_proj", "model.layers.30.mlp.experts.78.gate_proj", "model.layers.30.mlp.experts.79.gate_proj", "model.layers.30.mlp.experts.80.gate_proj", "model.layers.30.mlp.experts.81.gate_proj", "model.layers.30.mlp.experts.82.gate_proj", "model.layers.30.mlp.experts.83.gate_proj", "model.layers.30.mlp.experts.84.gate_proj", "model.layers.30.mlp.experts.85.gate_proj", "model.layers.30.mlp.experts.86.gate_proj", "model.layers.30.mlp.experts.87.gate_proj", "model.layers.30.mlp.experts.88.gate_proj", "model.layers.30.mlp.experts.89.gate_proj", "model.layers.30.mlp.experts.90.gate_proj", "model.layers.30.mlp.experts.91.gate_proj", "model.layers.30.mlp.experts.92.gate_proj", "model.layers.30.mlp.experts.93.gate_proj", "model.layers.30.mlp.experts.94.gate_proj", "model.layers.30.mlp.experts.95.gate_proj", "model.layers.30.mlp.experts.96.gate_proj", "model.layers.30.mlp.experts.97.gate_proj", "model.layers.30.mlp.experts.98.gate_proj", "model.layers.30.mlp.experts.99.gate_proj", "model.layers.30.mlp.experts.100.gate_proj", "model.layers.30.mlp.experts.101.gate_proj", "model.layers.30.mlp.experts.102.gate_proj", "model.layers.30.mlp.experts.103.gate_proj", "model.layers.30.mlp.experts.104.gate_proj", "model.layers.30.mlp.experts.105.gate_proj", "model.layers.30.mlp.experts.106.gate_proj", "model.layers.30.mlp.experts.107.gate_proj", "model.layers.30.mlp.experts.108.gate_proj", "model.layers.30.mlp.experts.109.gate_proj", "model.layers.30.mlp.experts.110.gate_proj", "model.layers.30.mlp.experts.111.gate_proj", "model.layers.30.mlp.experts.112.gate_proj", "model.layers.30.mlp.experts.113.gate_proj", "model.layers.30.mlp.experts.114.gate_proj", "model.layers.30.mlp.experts.115.gate_proj", "model.layers.30.mlp.experts.116.gate_proj", "model.layers.30.mlp.experts.117.gate_proj", "model.layers.30.mlp.experts.118.gate_proj", "model.layers.30.mlp.experts.119.gate_proj", "model.layers.30.mlp.experts.120.gate_proj", "model.layers.30.mlp.experts.121.gate_proj", "model.layers.30.mlp.experts.122.gate_proj", "model.layers.30.mlp.experts.123.gate_proj", "model.layers.30.mlp.experts.124.gate_proj", "model.layers.30.mlp.experts.125.gate_proj", "model.layers.30.mlp.experts.126.gate_proj", "model.layers.30.mlp.experts.127.gate_proj", "model.layers.30.mlp.experts.128.gate_proj", "model.layers.30.mlp.experts.129.gate_proj", "model.layers.30.mlp.experts.130.gate_proj", "model.layers.30.mlp.experts.131.gate_proj", "model.layers.30.mlp.experts.132.gate_proj", "model.layers.30.mlp.experts.133.gate_proj", "model.layers.30.mlp.experts.134.gate_proj", "model.layers.30.mlp.experts.135.gate_proj", "model.layers.30.mlp.experts.136.gate_proj", "model.layers.30.mlp.experts.137.gate_proj", "model.layers.30.mlp.experts.138.gate_proj", "model.layers.30.mlp.experts.139.gate_proj", "model.layers.30.mlp.experts.140.gate_proj", "model.layers.30.mlp.experts.141.gate_proj", "model.layers.30.mlp.experts.142.gate_proj", "model.layers.30.mlp.experts.143.gate_proj", "model.layers.30.mlp.experts.144.gate_proj", "model.layers.30.mlp.experts.145.gate_proj", "model.layers.30.mlp.experts.146.gate_proj", "model.layers.30.mlp.experts.147.gate_proj", "model.layers.30.mlp.experts.148.gate_proj", "model.layers.30.mlp.experts.149.gate_proj", "model.layers.30.mlp.experts.150.gate_proj", "model.layers.30.mlp.experts.151.gate_proj", "model.layers.30.mlp.experts.152.gate_proj", "model.layers.30.mlp.experts.153.gate_proj", "model.layers.30.mlp.experts.154.gate_proj", "model.layers.30.mlp.experts.155.gate_proj", "model.layers.30.mlp.experts.156.gate_proj", "model.layers.30.mlp.experts.157.gate_proj", "model.layers.30.mlp.experts.158.gate_proj", "model.layers.30.mlp.experts.159.gate_proj", "model.layers.30.mlp.experts.0.up_proj", "model.layers.30.mlp.experts.1.up_proj", "model.layers.30.mlp.experts.2.up_proj", "model.layers.30.mlp.experts.3.up_proj", "model.layers.30.mlp.experts.4.up_proj", "model.layers.30.mlp.experts.5.up_proj", "model.layers.30.mlp.experts.6.up_proj", "model.layers.30.mlp.experts.7.up_proj", "model.layers.30.mlp.experts.8.up_proj", "model.layers.30.mlp.experts.9.up_proj", "model.layers.30.mlp.experts.10.up_proj", "model.layers.30.mlp.experts.11.up_proj", "model.layers.30.mlp.experts.12.up_proj", "model.layers.30.mlp.experts.13.up_proj", "model.layers.30.mlp.experts.14.up_proj", "model.layers.30.mlp.experts.15.up_proj", "model.layers.30.mlp.experts.16.up_proj", "model.layers.30.mlp.experts.17.up_proj", "model.layers.30.mlp.experts.18.up_proj", "model.layers.30.mlp.experts.19.up_proj", "model.layers.30.mlp.experts.20.up_proj", "model.layers.30.mlp.experts.21.up_proj", "model.layers.30.mlp.experts.22.up_proj", "model.layers.30.mlp.experts.23.up_proj", "model.layers.30.mlp.experts.24.up_proj", "model.layers.30.mlp.experts.25.up_proj", "model.layers.30.mlp.experts.26.up_proj", "model.layers.30.mlp.experts.27.up_proj", "model.layers.30.mlp.experts.28.up_proj", "model.layers.30.mlp.experts.29.up_proj", "model.layers.30.mlp.experts.30.up_proj", "model.layers.30.mlp.experts.31.up_proj", "model.layers.30.mlp.experts.32.up_proj", "model.layers.30.mlp.experts.33.up_proj", "model.layers.30.mlp.experts.34.up_proj", "model.layers.30.mlp.experts.35.up_proj", "model.layers.30.mlp.experts.36.up_proj", "model.layers.30.mlp.experts.37.up_proj", "model.layers.30.mlp.experts.38.up_proj", "model.layers.30.mlp.experts.39.up_proj", "model.layers.30.mlp.experts.40.up_proj", "model.layers.30.mlp.experts.41.up_proj", "model.layers.30.mlp.experts.42.up_proj", "model.layers.30.mlp.experts.43.up_proj", "model.layers.30.mlp.experts.44.up_proj", "model.layers.30.mlp.experts.45.up_proj", "model.layers.30.mlp.experts.46.up_proj", "model.layers.30.mlp.experts.47.up_proj", "model.layers.30.mlp.experts.48.up_proj", "model.layers.30.mlp.experts.49.up_proj", "model.layers.30.mlp.experts.50.up_proj", "model.layers.30.mlp.experts.51.up_proj", "model.layers.30.mlp.experts.52.up_proj", "model.layers.30.mlp.experts.53.up_proj", "model.layers.30.mlp.experts.54.up_proj", "model.layers.30.mlp.experts.55.up_proj", "model.layers.30.mlp.experts.56.up_proj", "model.layers.30.mlp.experts.57.up_proj", "model.layers.30.mlp.experts.58.up_proj", "model.layers.30.mlp.experts.59.up_proj", "model.layers.30.mlp.experts.60.up_proj", "model.layers.30.mlp.experts.61.up_proj", "model.layers.30.mlp.experts.62.up_proj", "model.layers.30.mlp.experts.63.up_proj", "model.layers.30.mlp.experts.64.up_proj", "model.layers.30.mlp.experts.65.up_proj", "model.layers.30.mlp.experts.66.up_proj", "model.layers.30.mlp.experts.67.up_proj", "model.layers.30.mlp.experts.68.up_proj", "model.layers.30.mlp.experts.69.up_proj", "model.layers.30.mlp.experts.70.up_proj", "model.layers.30.mlp.experts.71.up_proj", "model.layers.30.mlp.experts.72.up_proj", "model.layers.30.mlp.experts.73.up_proj", "model.layers.30.mlp.experts.74.up_proj", "model.layers.30.mlp.experts.75.up_proj", "model.layers.30.mlp.experts.76.up_proj", "model.layers.30.mlp.experts.77.up_proj", "model.layers.30.mlp.experts.78.up_proj", "model.layers.30.mlp.experts.79.up_proj", "model.layers.30.mlp.experts.80.up_proj", "model.layers.30.mlp.experts.81.up_proj", "model.layers.30.mlp.experts.82.up_proj", "model.layers.30.mlp.experts.83.up_proj", "model.layers.30.mlp.experts.84.up_proj", "model.layers.30.mlp.experts.85.up_proj", "model.layers.30.mlp.experts.86.up_proj", "model.layers.30.mlp.experts.87.up_proj", "model.layers.30.mlp.experts.88.up_proj", "model.layers.30.mlp.experts.89.up_proj", "model.layers.30.mlp.experts.90.up_proj", "model.layers.30.mlp.experts.91.up_proj", "model.layers.30.mlp.experts.92.up_proj", "model.layers.30.mlp.experts.93.up_proj", "model.layers.30.mlp.experts.94.up_proj", "model.layers.30.mlp.experts.95.up_proj", "model.layers.30.mlp.experts.96.up_proj", "model.layers.30.mlp.experts.97.up_proj", "model.layers.30.mlp.experts.98.up_proj", "model.layers.30.mlp.experts.99.up_proj", "model.layers.30.mlp.experts.100.up_proj", "model.layers.30.mlp.experts.101.up_proj", "model.layers.30.mlp.experts.102.up_proj", "model.layers.30.mlp.experts.103.up_proj", "model.layers.30.mlp.experts.104.up_proj", "model.layers.30.mlp.experts.105.up_proj", "model.layers.30.mlp.experts.106.up_proj", "model.layers.30.mlp.experts.107.up_proj", "model.layers.30.mlp.experts.108.up_proj", "model.layers.30.mlp.experts.109.up_proj", "model.layers.30.mlp.experts.110.up_proj", "model.layers.30.mlp.experts.111.up_proj", "model.layers.30.mlp.experts.112.up_proj", "model.layers.30.mlp.experts.113.up_proj", "model.layers.30.mlp.experts.114.up_proj", "model.layers.30.mlp.experts.115.up_proj", "model.layers.30.mlp.experts.116.up_proj", "model.layers.30.mlp.experts.117.up_proj", "model.layers.30.mlp.experts.118.up_proj", "model.layers.30.mlp.experts.119.up_proj", "model.layers.30.mlp.experts.120.up_proj", "model.layers.30.mlp.experts.121.up_proj", "model.layers.30.mlp.experts.122.up_proj", "model.layers.30.mlp.experts.123.up_proj", "model.layers.30.mlp.experts.124.up_proj", "model.layers.30.mlp.experts.125.up_proj", "model.layers.30.mlp.experts.126.up_proj", "model.layers.30.mlp.experts.127.up_proj", "model.layers.30.mlp.experts.128.up_proj", "model.layers.30.mlp.experts.129.up_proj", "model.layers.30.mlp.experts.130.up_proj", "model.layers.30.mlp.experts.131.up_proj", "model.layers.30.mlp.experts.132.up_proj", "model.layers.30.mlp.experts.133.up_proj", "model.layers.30.mlp.experts.134.up_proj", "model.layers.30.mlp.experts.135.up_proj", "model.layers.30.mlp.experts.136.up_proj", "model.layers.30.mlp.experts.137.up_proj", "model.layers.30.mlp.experts.138.up_proj", "model.layers.30.mlp.experts.139.up_proj", "model.layers.30.mlp.experts.140.up_proj", "model.layers.30.mlp.experts.141.up_proj", "model.layers.30.mlp.experts.142.up_proj", "model.layers.30.mlp.experts.143.up_proj", "model.layers.30.mlp.experts.144.up_proj", "model.layers.30.mlp.experts.145.up_proj", "model.layers.30.mlp.experts.146.up_proj", "model.layers.30.mlp.experts.147.up_proj", "model.layers.30.mlp.experts.148.up_proj", "model.layers.30.mlp.experts.149.up_proj", "model.layers.30.mlp.experts.150.up_proj", "model.layers.30.mlp.experts.151.up_proj", "model.layers.30.mlp.experts.152.up_proj", "model.layers.30.mlp.experts.153.up_proj", "model.layers.30.mlp.experts.154.up_proj", "model.layers.30.mlp.experts.155.up_proj", "model.layers.30.mlp.experts.156.up_proj", "model.layers.30.mlp.experts.157.up_proj", "model.layers.30.mlp.experts.158.up_proj", "model.layers.30.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.8099475204944827e-06, "dbits": 5033164800 } ] }, { "idx": 182, "layers": [ "model.layers.30.mlp.experts.0.down_proj", "model.layers.30.mlp.experts.1.down_proj", "model.layers.30.mlp.experts.2.down_proj", "model.layers.30.mlp.experts.3.down_proj", "model.layers.30.mlp.experts.4.down_proj", "model.layers.30.mlp.experts.5.down_proj", "model.layers.30.mlp.experts.6.down_proj", "model.layers.30.mlp.experts.7.down_proj", "model.layers.30.mlp.experts.8.down_proj", "model.layers.30.mlp.experts.9.down_proj", "model.layers.30.mlp.experts.10.down_proj", "model.layers.30.mlp.experts.11.down_proj", "model.layers.30.mlp.experts.12.down_proj", "model.layers.30.mlp.experts.13.down_proj", "model.layers.30.mlp.experts.14.down_proj", "model.layers.30.mlp.experts.15.down_proj", "model.layers.30.mlp.experts.16.down_proj", "model.layers.30.mlp.experts.17.down_proj", "model.layers.30.mlp.experts.18.down_proj", "model.layers.30.mlp.experts.19.down_proj", "model.layers.30.mlp.experts.20.down_proj", "model.layers.30.mlp.experts.21.down_proj", "model.layers.30.mlp.experts.22.down_proj", "model.layers.30.mlp.experts.23.down_proj", "model.layers.30.mlp.experts.24.down_proj", "model.layers.30.mlp.experts.25.down_proj", "model.layers.30.mlp.experts.26.down_proj", "model.layers.30.mlp.experts.27.down_proj", "model.layers.30.mlp.experts.28.down_proj", "model.layers.30.mlp.experts.29.down_proj", "model.layers.30.mlp.experts.30.down_proj", "model.layers.30.mlp.experts.31.down_proj", "model.layers.30.mlp.experts.32.down_proj", "model.layers.30.mlp.experts.33.down_proj", "model.layers.30.mlp.experts.34.down_proj", "model.layers.30.mlp.experts.35.down_proj", "model.layers.30.mlp.experts.36.down_proj", "model.layers.30.mlp.experts.37.down_proj", "model.layers.30.mlp.experts.38.down_proj", "model.layers.30.mlp.experts.39.down_proj", "model.layers.30.mlp.experts.40.down_proj", "model.layers.30.mlp.experts.41.down_proj", "model.layers.30.mlp.experts.42.down_proj", "model.layers.30.mlp.experts.43.down_proj", "model.layers.30.mlp.experts.44.down_proj", "model.layers.30.mlp.experts.45.down_proj", "model.layers.30.mlp.experts.46.down_proj", "model.layers.30.mlp.experts.47.down_proj", "model.layers.30.mlp.experts.48.down_proj", "model.layers.30.mlp.experts.49.down_proj", "model.layers.30.mlp.experts.50.down_proj", "model.layers.30.mlp.experts.51.down_proj", "model.layers.30.mlp.experts.52.down_proj", "model.layers.30.mlp.experts.53.down_proj", "model.layers.30.mlp.experts.54.down_proj", "model.layers.30.mlp.experts.55.down_proj", "model.layers.30.mlp.experts.56.down_proj", "model.layers.30.mlp.experts.57.down_proj", "model.layers.30.mlp.experts.58.down_proj", "model.layers.30.mlp.experts.59.down_proj", "model.layers.30.mlp.experts.60.down_proj", "model.layers.30.mlp.experts.61.down_proj", "model.layers.30.mlp.experts.62.down_proj", "model.layers.30.mlp.experts.63.down_proj", "model.layers.30.mlp.experts.64.down_proj", "model.layers.30.mlp.experts.65.down_proj", "model.layers.30.mlp.experts.66.down_proj", "model.layers.30.mlp.experts.67.down_proj", "model.layers.30.mlp.experts.68.down_proj", "model.layers.30.mlp.experts.69.down_proj", "model.layers.30.mlp.experts.70.down_proj", "model.layers.30.mlp.experts.71.down_proj", "model.layers.30.mlp.experts.72.down_proj", "model.layers.30.mlp.experts.73.down_proj", "model.layers.30.mlp.experts.74.down_proj", "model.layers.30.mlp.experts.75.down_proj", "model.layers.30.mlp.experts.76.down_proj", "model.layers.30.mlp.experts.77.down_proj", "model.layers.30.mlp.experts.78.down_proj", "model.layers.30.mlp.experts.79.down_proj", "model.layers.30.mlp.experts.80.down_proj", "model.layers.30.mlp.experts.81.down_proj", "model.layers.30.mlp.experts.82.down_proj", "model.layers.30.mlp.experts.83.down_proj", "model.layers.30.mlp.experts.84.down_proj", "model.layers.30.mlp.experts.85.down_proj", "model.layers.30.mlp.experts.86.down_proj", "model.layers.30.mlp.experts.87.down_proj", "model.layers.30.mlp.experts.88.down_proj", "model.layers.30.mlp.experts.89.down_proj", "model.layers.30.mlp.experts.90.down_proj", "model.layers.30.mlp.experts.91.down_proj", "model.layers.30.mlp.experts.92.down_proj", "model.layers.30.mlp.experts.93.down_proj", "model.layers.30.mlp.experts.94.down_proj", "model.layers.30.mlp.experts.95.down_proj", "model.layers.30.mlp.experts.96.down_proj", "model.layers.30.mlp.experts.97.down_proj", "model.layers.30.mlp.experts.98.down_proj", "model.layers.30.mlp.experts.99.down_proj", "model.layers.30.mlp.experts.100.down_proj", "model.layers.30.mlp.experts.101.down_proj", "model.layers.30.mlp.experts.102.down_proj", "model.layers.30.mlp.experts.103.down_proj", "model.layers.30.mlp.experts.104.down_proj", "model.layers.30.mlp.experts.105.down_proj", "model.layers.30.mlp.experts.106.down_proj", "model.layers.30.mlp.experts.107.down_proj", "model.layers.30.mlp.experts.108.down_proj", "model.layers.30.mlp.experts.109.down_proj", "model.layers.30.mlp.experts.110.down_proj", "model.layers.30.mlp.experts.111.down_proj", "model.layers.30.mlp.experts.112.down_proj", "model.layers.30.mlp.experts.113.down_proj", "model.layers.30.mlp.experts.114.down_proj", "model.layers.30.mlp.experts.115.down_proj", "model.layers.30.mlp.experts.116.down_proj", "model.layers.30.mlp.experts.117.down_proj", "model.layers.30.mlp.experts.118.down_proj", "model.layers.30.mlp.experts.119.down_proj", "model.layers.30.mlp.experts.120.down_proj", "model.layers.30.mlp.experts.121.down_proj", "model.layers.30.mlp.experts.122.down_proj", "model.layers.30.mlp.experts.123.down_proj", "model.layers.30.mlp.experts.124.down_proj", "model.layers.30.mlp.experts.125.down_proj", "model.layers.30.mlp.experts.126.down_proj", "model.layers.30.mlp.experts.127.down_proj", "model.layers.30.mlp.experts.128.down_proj", "model.layers.30.mlp.experts.129.down_proj", "model.layers.30.mlp.experts.130.down_proj", "model.layers.30.mlp.experts.131.down_proj", "model.layers.30.mlp.experts.132.down_proj", "model.layers.30.mlp.experts.133.down_proj", "model.layers.30.mlp.experts.134.down_proj", "model.layers.30.mlp.experts.135.down_proj", "model.layers.30.mlp.experts.136.down_proj", "model.layers.30.mlp.experts.137.down_proj", "model.layers.30.mlp.experts.138.down_proj", "model.layers.30.mlp.experts.139.down_proj", "model.layers.30.mlp.experts.140.down_proj", "model.layers.30.mlp.experts.141.down_proj", "model.layers.30.mlp.experts.142.down_proj", "model.layers.30.mlp.experts.143.down_proj", "model.layers.30.mlp.experts.144.down_proj", "model.layers.30.mlp.experts.145.down_proj", "model.layers.30.mlp.experts.146.down_proj", "model.layers.30.mlp.experts.147.down_proj", "model.layers.30.mlp.experts.148.down_proj", "model.layers.30.mlp.experts.149.down_proj", "model.layers.30.mlp.experts.150.down_proj", "model.layers.30.mlp.experts.151.down_proj", "model.layers.30.mlp.experts.152.down_proj", "model.layers.30.mlp.experts.153.down_proj", "model.layers.30.mlp.experts.154.down_proj", "model.layers.30.mlp.experts.155.down_proj", "model.layers.30.mlp.experts.156.down_proj", "model.layers.30.mlp.experts.157.down_proj", "model.layers.30.mlp.experts.158.down_proj", "model.layers.30.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.479656202718659e-06, "dbits": 2516582400 } ] }, { "idx": 183, "layers": [ "model.layers.31.self_attn.q_proj" ], "candidates": [ { "dkld": 5.766798858530906e-06, "dbits": 125829120 } ] }, { "idx": 184, "layers": [ "model.layers.31.self_attn.k_proj", "model.layers.31.self_attn.v_proj" ], "candidates": [ { "dkld": -3.6689423723145886e-06, "dbits": 20971520 } ] }, { "idx": 185, "layers": [ "model.layers.31.self_attn.o_proj" ], "candidates": [ { "dkld": -7.807053043507061e-06, "dbits": 125829120 } ] }, { "idx": 186, "layers": [ "model.layers.31.mlp.shared_experts.gate_proj", "model.layers.31.mlp.shared_experts.up_proj", "model.layers.31.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.3787852367386005e-06, "dbits": 47185920 } ] }, { "idx": 187, "layers": [ "model.layers.31.mlp.experts.0.gate_proj", "model.layers.31.mlp.experts.1.gate_proj", "model.layers.31.mlp.experts.2.gate_proj", "model.layers.31.mlp.experts.3.gate_proj", "model.layers.31.mlp.experts.4.gate_proj", "model.layers.31.mlp.experts.5.gate_proj", "model.layers.31.mlp.experts.6.gate_proj", "model.layers.31.mlp.experts.7.gate_proj", "model.layers.31.mlp.experts.8.gate_proj", "model.layers.31.mlp.experts.9.gate_proj", "model.layers.31.mlp.experts.10.gate_proj", "model.layers.31.mlp.experts.11.gate_proj", "model.layers.31.mlp.experts.12.gate_proj", "model.layers.31.mlp.experts.13.gate_proj", "model.layers.31.mlp.experts.14.gate_proj", "model.layers.31.mlp.experts.15.gate_proj", "model.layers.31.mlp.experts.16.gate_proj", "model.layers.31.mlp.experts.17.gate_proj", "model.layers.31.mlp.experts.18.gate_proj", "model.layers.31.mlp.experts.19.gate_proj", "model.layers.31.mlp.experts.20.gate_proj", "model.layers.31.mlp.experts.21.gate_proj", "model.layers.31.mlp.experts.22.gate_proj", "model.layers.31.mlp.experts.23.gate_proj", "model.layers.31.mlp.experts.24.gate_proj", "model.layers.31.mlp.experts.25.gate_proj", "model.layers.31.mlp.experts.26.gate_proj", "model.layers.31.mlp.experts.27.gate_proj", "model.layers.31.mlp.experts.28.gate_proj", "model.layers.31.mlp.experts.29.gate_proj", "model.layers.31.mlp.experts.30.gate_proj", "model.layers.31.mlp.experts.31.gate_proj", "model.layers.31.mlp.experts.32.gate_proj", "model.layers.31.mlp.experts.33.gate_proj", "model.layers.31.mlp.experts.34.gate_proj", "model.layers.31.mlp.experts.35.gate_proj", "model.layers.31.mlp.experts.36.gate_proj", "model.layers.31.mlp.experts.37.gate_proj", "model.layers.31.mlp.experts.38.gate_proj", "model.layers.31.mlp.experts.39.gate_proj", "model.layers.31.mlp.experts.40.gate_proj", "model.layers.31.mlp.experts.41.gate_proj", "model.layers.31.mlp.experts.42.gate_proj", "model.layers.31.mlp.experts.43.gate_proj", "model.layers.31.mlp.experts.44.gate_proj", "model.layers.31.mlp.experts.45.gate_proj", "model.layers.31.mlp.experts.46.gate_proj", "model.layers.31.mlp.experts.47.gate_proj", "model.layers.31.mlp.experts.48.gate_proj", "model.layers.31.mlp.experts.49.gate_proj", "model.layers.31.mlp.experts.50.gate_proj", "model.layers.31.mlp.experts.51.gate_proj", "model.layers.31.mlp.experts.52.gate_proj", "model.layers.31.mlp.experts.53.gate_proj", "model.layers.31.mlp.experts.54.gate_proj", "model.layers.31.mlp.experts.55.gate_proj", "model.layers.31.mlp.experts.56.gate_proj", "model.layers.31.mlp.experts.57.gate_proj", "model.layers.31.mlp.experts.58.gate_proj", "model.layers.31.mlp.experts.59.gate_proj", "model.layers.31.mlp.experts.60.gate_proj", "model.layers.31.mlp.experts.61.gate_proj", "model.layers.31.mlp.experts.62.gate_proj", "model.layers.31.mlp.experts.63.gate_proj", "model.layers.31.mlp.experts.64.gate_proj", "model.layers.31.mlp.experts.65.gate_proj", "model.layers.31.mlp.experts.66.gate_proj", "model.layers.31.mlp.experts.67.gate_proj", "model.layers.31.mlp.experts.68.gate_proj", "model.layers.31.mlp.experts.69.gate_proj", "model.layers.31.mlp.experts.70.gate_proj", "model.layers.31.mlp.experts.71.gate_proj", "model.layers.31.mlp.experts.72.gate_proj", "model.layers.31.mlp.experts.73.gate_proj", "model.layers.31.mlp.experts.74.gate_proj", "model.layers.31.mlp.experts.75.gate_proj", "model.layers.31.mlp.experts.76.gate_proj", "model.layers.31.mlp.experts.77.gate_proj", "model.layers.31.mlp.experts.78.gate_proj", "model.layers.31.mlp.experts.79.gate_proj", "model.layers.31.mlp.experts.80.gate_proj", "model.layers.31.mlp.experts.81.gate_proj", "model.layers.31.mlp.experts.82.gate_proj", "model.layers.31.mlp.experts.83.gate_proj", "model.layers.31.mlp.experts.84.gate_proj", "model.layers.31.mlp.experts.85.gate_proj", "model.layers.31.mlp.experts.86.gate_proj", "model.layers.31.mlp.experts.87.gate_proj", "model.layers.31.mlp.experts.88.gate_proj", "model.layers.31.mlp.experts.89.gate_proj", "model.layers.31.mlp.experts.90.gate_proj", "model.layers.31.mlp.experts.91.gate_proj", "model.layers.31.mlp.experts.92.gate_proj", "model.layers.31.mlp.experts.93.gate_proj", "model.layers.31.mlp.experts.94.gate_proj", "model.layers.31.mlp.experts.95.gate_proj", "model.layers.31.mlp.experts.96.gate_proj", "model.layers.31.mlp.experts.97.gate_proj", "model.layers.31.mlp.experts.98.gate_proj", "model.layers.31.mlp.experts.99.gate_proj", "model.layers.31.mlp.experts.100.gate_proj", "model.layers.31.mlp.experts.101.gate_proj", "model.layers.31.mlp.experts.102.gate_proj", "model.layers.31.mlp.experts.103.gate_proj", "model.layers.31.mlp.experts.104.gate_proj", "model.layers.31.mlp.experts.105.gate_proj", "model.layers.31.mlp.experts.106.gate_proj", "model.layers.31.mlp.experts.107.gate_proj", "model.layers.31.mlp.experts.108.gate_proj", "model.layers.31.mlp.experts.109.gate_proj", "model.layers.31.mlp.experts.110.gate_proj", "model.layers.31.mlp.experts.111.gate_proj", "model.layers.31.mlp.experts.112.gate_proj", "model.layers.31.mlp.experts.113.gate_proj", "model.layers.31.mlp.experts.114.gate_proj", "model.layers.31.mlp.experts.115.gate_proj", "model.layers.31.mlp.experts.116.gate_proj", "model.layers.31.mlp.experts.117.gate_proj", "model.layers.31.mlp.experts.118.gate_proj", "model.layers.31.mlp.experts.119.gate_proj", "model.layers.31.mlp.experts.120.gate_proj", "model.layers.31.mlp.experts.121.gate_proj", "model.layers.31.mlp.experts.122.gate_proj", "model.layers.31.mlp.experts.123.gate_proj", "model.layers.31.mlp.experts.124.gate_proj", "model.layers.31.mlp.experts.125.gate_proj", "model.layers.31.mlp.experts.126.gate_proj", "model.layers.31.mlp.experts.127.gate_proj", "model.layers.31.mlp.experts.128.gate_proj", "model.layers.31.mlp.experts.129.gate_proj", "model.layers.31.mlp.experts.130.gate_proj", "model.layers.31.mlp.experts.131.gate_proj", "model.layers.31.mlp.experts.132.gate_proj", "model.layers.31.mlp.experts.133.gate_proj", "model.layers.31.mlp.experts.134.gate_proj", "model.layers.31.mlp.experts.135.gate_proj", "model.layers.31.mlp.experts.136.gate_proj", "model.layers.31.mlp.experts.137.gate_proj", "model.layers.31.mlp.experts.138.gate_proj", "model.layers.31.mlp.experts.139.gate_proj", "model.layers.31.mlp.experts.140.gate_proj", "model.layers.31.mlp.experts.141.gate_proj", "model.layers.31.mlp.experts.142.gate_proj", "model.layers.31.mlp.experts.143.gate_proj", "model.layers.31.mlp.experts.144.gate_proj", "model.layers.31.mlp.experts.145.gate_proj", "model.layers.31.mlp.experts.146.gate_proj", "model.layers.31.mlp.experts.147.gate_proj", "model.layers.31.mlp.experts.148.gate_proj", "model.layers.31.mlp.experts.149.gate_proj", "model.layers.31.mlp.experts.150.gate_proj", "model.layers.31.mlp.experts.151.gate_proj", "model.layers.31.mlp.experts.152.gate_proj", "model.layers.31.mlp.experts.153.gate_proj", "model.layers.31.mlp.experts.154.gate_proj", "model.layers.31.mlp.experts.155.gate_proj", "model.layers.31.mlp.experts.156.gate_proj", "model.layers.31.mlp.experts.157.gate_proj", "model.layers.31.mlp.experts.158.gate_proj", "model.layers.31.mlp.experts.159.gate_proj", "model.layers.31.mlp.experts.0.up_proj", "model.layers.31.mlp.experts.1.up_proj", "model.layers.31.mlp.experts.2.up_proj", "model.layers.31.mlp.experts.3.up_proj", "model.layers.31.mlp.experts.4.up_proj", "model.layers.31.mlp.experts.5.up_proj", "model.layers.31.mlp.experts.6.up_proj", "model.layers.31.mlp.experts.7.up_proj", "model.layers.31.mlp.experts.8.up_proj", "model.layers.31.mlp.experts.9.up_proj", "model.layers.31.mlp.experts.10.up_proj", "model.layers.31.mlp.experts.11.up_proj", "model.layers.31.mlp.experts.12.up_proj", "model.layers.31.mlp.experts.13.up_proj", "model.layers.31.mlp.experts.14.up_proj", "model.layers.31.mlp.experts.15.up_proj", "model.layers.31.mlp.experts.16.up_proj", "model.layers.31.mlp.experts.17.up_proj", "model.layers.31.mlp.experts.18.up_proj", "model.layers.31.mlp.experts.19.up_proj", "model.layers.31.mlp.experts.20.up_proj", "model.layers.31.mlp.experts.21.up_proj", "model.layers.31.mlp.experts.22.up_proj", "model.layers.31.mlp.experts.23.up_proj", "model.layers.31.mlp.experts.24.up_proj", "model.layers.31.mlp.experts.25.up_proj", "model.layers.31.mlp.experts.26.up_proj", "model.layers.31.mlp.experts.27.up_proj", "model.layers.31.mlp.experts.28.up_proj", "model.layers.31.mlp.experts.29.up_proj", "model.layers.31.mlp.experts.30.up_proj", "model.layers.31.mlp.experts.31.up_proj", "model.layers.31.mlp.experts.32.up_proj", "model.layers.31.mlp.experts.33.up_proj", "model.layers.31.mlp.experts.34.up_proj", "model.layers.31.mlp.experts.35.up_proj", "model.layers.31.mlp.experts.36.up_proj", "model.layers.31.mlp.experts.37.up_proj", "model.layers.31.mlp.experts.38.up_proj", "model.layers.31.mlp.experts.39.up_proj", "model.layers.31.mlp.experts.40.up_proj", "model.layers.31.mlp.experts.41.up_proj", "model.layers.31.mlp.experts.42.up_proj", "model.layers.31.mlp.experts.43.up_proj", "model.layers.31.mlp.experts.44.up_proj", "model.layers.31.mlp.experts.45.up_proj", "model.layers.31.mlp.experts.46.up_proj", "model.layers.31.mlp.experts.47.up_proj", "model.layers.31.mlp.experts.48.up_proj", "model.layers.31.mlp.experts.49.up_proj", "model.layers.31.mlp.experts.50.up_proj", "model.layers.31.mlp.experts.51.up_proj", "model.layers.31.mlp.experts.52.up_proj", "model.layers.31.mlp.experts.53.up_proj", "model.layers.31.mlp.experts.54.up_proj", "model.layers.31.mlp.experts.55.up_proj", "model.layers.31.mlp.experts.56.up_proj", "model.layers.31.mlp.experts.57.up_proj", "model.layers.31.mlp.experts.58.up_proj", "model.layers.31.mlp.experts.59.up_proj", "model.layers.31.mlp.experts.60.up_proj", "model.layers.31.mlp.experts.61.up_proj", "model.layers.31.mlp.experts.62.up_proj", "model.layers.31.mlp.experts.63.up_proj", "model.layers.31.mlp.experts.64.up_proj", "model.layers.31.mlp.experts.65.up_proj", "model.layers.31.mlp.experts.66.up_proj", "model.layers.31.mlp.experts.67.up_proj", "model.layers.31.mlp.experts.68.up_proj", "model.layers.31.mlp.experts.69.up_proj", "model.layers.31.mlp.experts.70.up_proj", "model.layers.31.mlp.experts.71.up_proj", "model.layers.31.mlp.experts.72.up_proj", "model.layers.31.mlp.experts.73.up_proj", "model.layers.31.mlp.experts.74.up_proj", "model.layers.31.mlp.experts.75.up_proj", "model.layers.31.mlp.experts.76.up_proj", "model.layers.31.mlp.experts.77.up_proj", "model.layers.31.mlp.experts.78.up_proj", "model.layers.31.mlp.experts.79.up_proj", "model.layers.31.mlp.experts.80.up_proj", "model.layers.31.mlp.experts.81.up_proj", "model.layers.31.mlp.experts.82.up_proj", "model.layers.31.mlp.experts.83.up_proj", "model.layers.31.mlp.experts.84.up_proj", "model.layers.31.mlp.experts.85.up_proj", "model.layers.31.mlp.experts.86.up_proj", "model.layers.31.mlp.experts.87.up_proj", "model.layers.31.mlp.experts.88.up_proj", "model.layers.31.mlp.experts.89.up_proj", "model.layers.31.mlp.experts.90.up_proj", "model.layers.31.mlp.experts.91.up_proj", "model.layers.31.mlp.experts.92.up_proj", "model.layers.31.mlp.experts.93.up_proj", "model.layers.31.mlp.experts.94.up_proj", "model.layers.31.mlp.experts.95.up_proj", "model.layers.31.mlp.experts.96.up_proj", "model.layers.31.mlp.experts.97.up_proj", "model.layers.31.mlp.experts.98.up_proj", "model.layers.31.mlp.experts.99.up_proj", "model.layers.31.mlp.experts.100.up_proj", "model.layers.31.mlp.experts.101.up_proj", "model.layers.31.mlp.experts.102.up_proj", "model.layers.31.mlp.experts.103.up_proj", "model.layers.31.mlp.experts.104.up_proj", "model.layers.31.mlp.experts.105.up_proj", "model.layers.31.mlp.experts.106.up_proj", "model.layers.31.mlp.experts.107.up_proj", "model.layers.31.mlp.experts.108.up_proj", "model.layers.31.mlp.experts.109.up_proj", "model.layers.31.mlp.experts.110.up_proj", "model.layers.31.mlp.experts.111.up_proj", "model.layers.31.mlp.experts.112.up_proj", "model.layers.31.mlp.experts.113.up_proj", "model.layers.31.mlp.experts.114.up_proj", "model.layers.31.mlp.experts.115.up_proj", "model.layers.31.mlp.experts.116.up_proj", "model.layers.31.mlp.experts.117.up_proj", "model.layers.31.mlp.experts.118.up_proj", "model.layers.31.mlp.experts.119.up_proj", "model.layers.31.mlp.experts.120.up_proj", "model.layers.31.mlp.experts.121.up_proj", "model.layers.31.mlp.experts.122.up_proj", "model.layers.31.mlp.experts.123.up_proj", "model.layers.31.mlp.experts.124.up_proj", "model.layers.31.mlp.experts.125.up_proj", "model.layers.31.mlp.experts.126.up_proj", "model.layers.31.mlp.experts.127.up_proj", "model.layers.31.mlp.experts.128.up_proj", "model.layers.31.mlp.experts.129.up_proj", "model.layers.31.mlp.experts.130.up_proj", "model.layers.31.mlp.experts.131.up_proj", "model.layers.31.mlp.experts.132.up_proj", "model.layers.31.mlp.experts.133.up_proj", "model.layers.31.mlp.experts.134.up_proj", "model.layers.31.mlp.experts.135.up_proj", "model.layers.31.mlp.experts.136.up_proj", "model.layers.31.mlp.experts.137.up_proj", "model.layers.31.mlp.experts.138.up_proj", "model.layers.31.mlp.experts.139.up_proj", "model.layers.31.mlp.experts.140.up_proj", "model.layers.31.mlp.experts.141.up_proj", "model.layers.31.mlp.experts.142.up_proj", "model.layers.31.mlp.experts.143.up_proj", "model.layers.31.mlp.experts.144.up_proj", "model.layers.31.mlp.experts.145.up_proj", "model.layers.31.mlp.experts.146.up_proj", "model.layers.31.mlp.experts.147.up_proj", "model.layers.31.mlp.experts.148.up_proj", "model.layers.31.mlp.experts.149.up_proj", "model.layers.31.mlp.experts.150.up_proj", "model.layers.31.mlp.experts.151.up_proj", "model.layers.31.mlp.experts.152.up_proj", "model.layers.31.mlp.experts.153.up_proj", "model.layers.31.mlp.experts.154.up_proj", "model.layers.31.mlp.experts.155.up_proj", "model.layers.31.mlp.experts.156.up_proj", "model.layers.31.mlp.experts.157.up_proj", "model.layers.31.mlp.experts.158.up_proj", "model.layers.31.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -5.293521098792119e-07, "dbits": 5033164800 } ] }, { "idx": 188, "layers": [ "model.layers.31.mlp.experts.0.down_proj", "model.layers.31.mlp.experts.1.down_proj", "model.layers.31.mlp.experts.2.down_proj", "model.layers.31.mlp.experts.3.down_proj", "model.layers.31.mlp.experts.4.down_proj", "model.layers.31.mlp.experts.5.down_proj", "model.layers.31.mlp.experts.6.down_proj", "model.layers.31.mlp.experts.7.down_proj", "model.layers.31.mlp.experts.8.down_proj", "model.layers.31.mlp.experts.9.down_proj", "model.layers.31.mlp.experts.10.down_proj", "model.layers.31.mlp.experts.11.down_proj", "model.layers.31.mlp.experts.12.down_proj", "model.layers.31.mlp.experts.13.down_proj", "model.layers.31.mlp.experts.14.down_proj", "model.layers.31.mlp.experts.15.down_proj", "model.layers.31.mlp.experts.16.down_proj", "model.layers.31.mlp.experts.17.down_proj", "model.layers.31.mlp.experts.18.down_proj", "model.layers.31.mlp.experts.19.down_proj", "model.layers.31.mlp.experts.20.down_proj", "model.layers.31.mlp.experts.21.down_proj", "model.layers.31.mlp.experts.22.down_proj", "model.layers.31.mlp.experts.23.down_proj", "model.layers.31.mlp.experts.24.down_proj", "model.layers.31.mlp.experts.25.down_proj", "model.layers.31.mlp.experts.26.down_proj", "model.layers.31.mlp.experts.27.down_proj", "model.layers.31.mlp.experts.28.down_proj", "model.layers.31.mlp.experts.29.down_proj", "model.layers.31.mlp.experts.30.down_proj", "model.layers.31.mlp.experts.31.down_proj", "model.layers.31.mlp.experts.32.down_proj", "model.layers.31.mlp.experts.33.down_proj", "model.layers.31.mlp.experts.34.down_proj", "model.layers.31.mlp.experts.35.down_proj", "model.layers.31.mlp.experts.36.down_proj", "model.layers.31.mlp.experts.37.down_proj", "model.layers.31.mlp.experts.38.down_proj", "model.layers.31.mlp.experts.39.down_proj", "model.layers.31.mlp.experts.40.down_proj", "model.layers.31.mlp.experts.41.down_proj", "model.layers.31.mlp.experts.42.down_proj", "model.layers.31.mlp.experts.43.down_proj", "model.layers.31.mlp.experts.44.down_proj", "model.layers.31.mlp.experts.45.down_proj", "model.layers.31.mlp.experts.46.down_proj", "model.layers.31.mlp.experts.47.down_proj", "model.layers.31.mlp.experts.48.down_proj", "model.layers.31.mlp.experts.49.down_proj", "model.layers.31.mlp.experts.50.down_proj", "model.layers.31.mlp.experts.51.down_proj", "model.layers.31.mlp.experts.52.down_proj", "model.layers.31.mlp.experts.53.down_proj", "model.layers.31.mlp.experts.54.down_proj", "model.layers.31.mlp.experts.55.down_proj", "model.layers.31.mlp.experts.56.down_proj", "model.layers.31.mlp.experts.57.down_proj", "model.layers.31.mlp.experts.58.down_proj", "model.layers.31.mlp.experts.59.down_proj", "model.layers.31.mlp.experts.60.down_proj", "model.layers.31.mlp.experts.61.down_proj", "model.layers.31.mlp.experts.62.down_proj", "model.layers.31.mlp.experts.63.down_proj", "model.layers.31.mlp.experts.64.down_proj", "model.layers.31.mlp.experts.65.down_proj", "model.layers.31.mlp.experts.66.down_proj", "model.layers.31.mlp.experts.67.down_proj", "model.layers.31.mlp.experts.68.down_proj", "model.layers.31.mlp.experts.69.down_proj", "model.layers.31.mlp.experts.70.down_proj", "model.layers.31.mlp.experts.71.down_proj", "model.layers.31.mlp.experts.72.down_proj", "model.layers.31.mlp.experts.73.down_proj", "model.layers.31.mlp.experts.74.down_proj", "model.layers.31.mlp.experts.75.down_proj", "model.layers.31.mlp.experts.76.down_proj", "model.layers.31.mlp.experts.77.down_proj", "model.layers.31.mlp.experts.78.down_proj", "model.layers.31.mlp.experts.79.down_proj", "model.layers.31.mlp.experts.80.down_proj", "model.layers.31.mlp.experts.81.down_proj", "model.layers.31.mlp.experts.82.down_proj", "model.layers.31.mlp.experts.83.down_proj", "model.layers.31.mlp.experts.84.down_proj", "model.layers.31.mlp.experts.85.down_proj", "model.layers.31.mlp.experts.86.down_proj", "model.layers.31.mlp.experts.87.down_proj", "model.layers.31.mlp.experts.88.down_proj", "model.layers.31.mlp.experts.89.down_proj", "model.layers.31.mlp.experts.90.down_proj", "model.layers.31.mlp.experts.91.down_proj", "model.layers.31.mlp.experts.92.down_proj", "model.layers.31.mlp.experts.93.down_proj", "model.layers.31.mlp.experts.94.down_proj", "model.layers.31.mlp.experts.95.down_proj", "model.layers.31.mlp.experts.96.down_proj", "model.layers.31.mlp.experts.97.down_proj", "model.layers.31.mlp.experts.98.down_proj", "model.layers.31.mlp.experts.99.down_proj", "model.layers.31.mlp.experts.100.down_proj", "model.layers.31.mlp.experts.101.down_proj", "model.layers.31.mlp.experts.102.down_proj", "model.layers.31.mlp.experts.103.down_proj", "model.layers.31.mlp.experts.104.down_proj", "model.layers.31.mlp.experts.105.down_proj", "model.layers.31.mlp.experts.106.down_proj", "model.layers.31.mlp.experts.107.down_proj", "model.layers.31.mlp.experts.108.down_proj", "model.layers.31.mlp.experts.109.down_proj", "model.layers.31.mlp.experts.110.down_proj", "model.layers.31.mlp.experts.111.down_proj", "model.layers.31.mlp.experts.112.down_proj", "model.layers.31.mlp.experts.113.down_proj", "model.layers.31.mlp.experts.114.down_proj", "model.layers.31.mlp.experts.115.down_proj", "model.layers.31.mlp.experts.116.down_proj", "model.layers.31.mlp.experts.117.down_proj", "model.layers.31.mlp.experts.118.down_proj", "model.layers.31.mlp.experts.119.down_proj", "model.layers.31.mlp.experts.120.down_proj", "model.layers.31.mlp.experts.121.down_proj", "model.layers.31.mlp.experts.122.down_proj", "model.layers.31.mlp.experts.123.down_proj", "model.layers.31.mlp.experts.124.down_proj", "model.layers.31.mlp.experts.125.down_proj", "model.layers.31.mlp.experts.126.down_proj", "model.layers.31.mlp.experts.127.down_proj", "model.layers.31.mlp.experts.128.down_proj", "model.layers.31.mlp.experts.129.down_proj", "model.layers.31.mlp.experts.130.down_proj", "model.layers.31.mlp.experts.131.down_proj", "model.layers.31.mlp.experts.132.down_proj", "model.layers.31.mlp.experts.133.down_proj", "model.layers.31.mlp.experts.134.down_proj", "model.layers.31.mlp.experts.135.down_proj", "model.layers.31.mlp.experts.136.down_proj", "model.layers.31.mlp.experts.137.down_proj", "model.layers.31.mlp.experts.138.down_proj", "model.layers.31.mlp.experts.139.down_proj", "model.layers.31.mlp.experts.140.down_proj", "model.layers.31.mlp.experts.141.down_proj", "model.layers.31.mlp.experts.142.down_proj", "model.layers.31.mlp.experts.143.down_proj", "model.layers.31.mlp.experts.144.down_proj", "model.layers.31.mlp.experts.145.down_proj", "model.layers.31.mlp.experts.146.down_proj", "model.layers.31.mlp.experts.147.down_proj", "model.layers.31.mlp.experts.148.down_proj", "model.layers.31.mlp.experts.149.down_proj", "model.layers.31.mlp.experts.150.down_proj", "model.layers.31.mlp.experts.151.down_proj", "model.layers.31.mlp.experts.152.down_proj", "model.layers.31.mlp.experts.153.down_proj", "model.layers.31.mlp.experts.154.down_proj", "model.layers.31.mlp.experts.155.down_proj", "model.layers.31.mlp.experts.156.down_proj", "model.layers.31.mlp.experts.157.down_proj", "model.layers.31.mlp.experts.158.down_proj", "model.layers.31.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.797002136706872e-07, "dbits": 2516582400 } ] }, { "idx": 189, "layers": [ "model.layers.32.self_attn.q_proj" ], "candidates": [ { "dkld": 3.847031621262431e-06, "dbits": 125829120 } ] }, { "idx": 190, "layers": [ "model.layers.32.self_attn.k_proj", "model.layers.32.self_attn.v_proj" ], "candidates": [ { "dkld": -1.0844098869711117e-05, "dbits": 20971520 } ] }, { "idx": 191, "layers": [ "model.layers.32.self_attn.o_proj" ], "candidates": [ { "dkld": -6.443343590945049e-06, "dbits": 125829120 } ] }, { "idx": 192, "layers": [ "model.layers.32.mlp.shared_experts.gate_proj", "model.layers.32.mlp.shared_experts.up_proj", "model.layers.32.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.300855941139162e-06, "dbits": 47185920 } ] }, { "idx": 193, "layers": [ "model.layers.32.mlp.experts.0.gate_proj", "model.layers.32.mlp.experts.1.gate_proj", "model.layers.32.mlp.experts.2.gate_proj", "model.layers.32.mlp.experts.3.gate_proj", "model.layers.32.mlp.experts.4.gate_proj", "model.layers.32.mlp.experts.5.gate_proj", "model.layers.32.mlp.experts.6.gate_proj", "model.layers.32.mlp.experts.7.gate_proj", "model.layers.32.mlp.experts.8.gate_proj", "model.layers.32.mlp.experts.9.gate_proj", "model.layers.32.mlp.experts.10.gate_proj", "model.layers.32.mlp.experts.11.gate_proj", "model.layers.32.mlp.experts.12.gate_proj", "model.layers.32.mlp.experts.13.gate_proj", "model.layers.32.mlp.experts.14.gate_proj", "model.layers.32.mlp.experts.15.gate_proj", "model.layers.32.mlp.experts.16.gate_proj", "model.layers.32.mlp.experts.17.gate_proj", "model.layers.32.mlp.experts.18.gate_proj", "model.layers.32.mlp.experts.19.gate_proj", "model.layers.32.mlp.experts.20.gate_proj", "model.layers.32.mlp.experts.21.gate_proj", "model.layers.32.mlp.experts.22.gate_proj", "model.layers.32.mlp.experts.23.gate_proj", "model.layers.32.mlp.experts.24.gate_proj", "model.layers.32.mlp.experts.25.gate_proj", "model.layers.32.mlp.experts.26.gate_proj", "model.layers.32.mlp.experts.27.gate_proj", "model.layers.32.mlp.experts.28.gate_proj", "model.layers.32.mlp.experts.29.gate_proj", "model.layers.32.mlp.experts.30.gate_proj", "model.layers.32.mlp.experts.31.gate_proj", "model.layers.32.mlp.experts.32.gate_proj", "model.layers.32.mlp.experts.33.gate_proj", "model.layers.32.mlp.experts.34.gate_proj", "model.layers.32.mlp.experts.35.gate_proj", "model.layers.32.mlp.experts.36.gate_proj", "model.layers.32.mlp.experts.37.gate_proj", "model.layers.32.mlp.experts.38.gate_proj", "model.layers.32.mlp.experts.39.gate_proj", "model.layers.32.mlp.experts.40.gate_proj", "model.layers.32.mlp.experts.41.gate_proj", "model.layers.32.mlp.experts.42.gate_proj", "model.layers.32.mlp.experts.43.gate_proj", "model.layers.32.mlp.experts.44.gate_proj", "model.layers.32.mlp.experts.45.gate_proj", "model.layers.32.mlp.experts.46.gate_proj", "model.layers.32.mlp.experts.47.gate_proj", "model.layers.32.mlp.experts.48.gate_proj", "model.layers.32.mlp.experts.49.gate_proj", "model.layers.32.mlp.experts.50.gate_proj", "model.layers.32.mlp.experts.51.gate_proj", "model.layers.32.mlp.experts.52.gate_proj", "model.layers.32.mlp.experts.53.gate_proj", "model.layers.32.mlp.experts.54.gate_proj", "model.layers.32.mlp.experts.55.gate_proj", "model.layers.32.mlp.experts.56.gate_proj", "model.layers.32.mlp.experts.57.gate_proj", "model.layers.32.mlp.experts.58.gate_proj", "model.layers.32.mlp.experts.59.gate_proj", "model.layers.32.mlp.experts.60.gate_proj", "model.layers.32.mlp.experts.61.gate_proj", "model.layers.32.mlp.experts.62.gate_proj", "model.layers.32.mlp.experts.63.gate_proj", "model.layers.32.mlp.experts.64.gate_proj", "model.layers.32.mlp.experts.65.gate_proj", "model.layers.32.mlp.experts.66.gate_proj", "model.layers.32.mlp.experts.67.gate_proj", "model.layers.32.mlp.experts.68.gate_proj", "model.layers.32.mlp.experts.69.gate_proj", "model.layers.32.mlp.experts.70.gate_proj", "model.layers.32.mlp.experts.71.gate_proj", "model.layers.32.mlp.experts.72.gate_proj", "model.layers.32.mlp.experts.73.gate_proj", "model.layers.32.mlp.experts.74.gate_proj", "model.layers.32.mlp.experts.75.gate_proj", "model.layers.32.mlp.experts.76.gate_proj", "model.layers.32.mlp.experts.77.gate_proj", "model.layers.32.mlp.experts.78.gate_proj", "model.layers.32.mlp.experts.79.gate_proj", "model.layers.32.mlp.experts.80.gate_proj", "model.layers.32.mlp.experts.81.gate_proj", "model.layers.32.mlp.experts.82.gate_proj", "model.layers.32.mlp.experts.83.gate_proj", "model.layers.32.mlp.experts.84.gate_proj", "model.layers.32.mlp.experts.85.gate_proj", "model.layers.32.mlp.experts.86.gate_proj", "model.layers.32.mlp.experts.87.gate_proj", "model.layers.32.mlp.experts.88.gate_proj", "model.layers.32.mlp.experts.89.gate_proj", "model.layers.32.mlp.experts.90.gate_proj", "model.layers.32.mlp.experts.91.gate_proj", "model.layers.32.mlp.experts.92.gate_proj", "model.layers.32.mlp.experts.93.gate_proj", "model.layers.32.mlp.experts.94.gate_proj", "model.layers.32.mlp.experts.95.gate_proj", "model.layers.32.mlp.experts.96.gate_proj", "model.layers.32.mlp.experts.97.gate_proj", "model.layers.32.mlp.experts.98.gate_proj", "model.layers.32.mlp.experts.99.gate_proj", "model.layers.32.mlp.experts.100.gate_proj", "model.layers.32.mlp.experts.101.gate_proj", "model.layers.32.mlp.experts.102.gate_proj", "model.layers.32.mlp.experts.103.gate_proj", "model.layers.32.mlp.experts.104.gate_proj", "model.layers.32.mlp.experts.105.gate_proj", "model.layers.32.mlp.experts.106.gate_proj", "model.layers.32.mlp.experts.107.gate_proj", "model.layers.32.mlp.experts.108.gate_proj", "model.layers.32.mlp.experts.109.gate_proj", "model.layers.32.mlp.experts.110.gate_proj", "model.layers.32.mlp.experts.111.gate_proj", "model.layers.32.mlp.experts.112.gate_proj", "model.layers.32.mlp.experts.113.gate_proj", "model.layers.32.mlp.experts.114.gate_proj", "model.layers.32.mlp.experts.115.gate_proj", "model.layers.32.mlp.experts.116.gate_proj", "model.layers.32.mlp.experts.117.gate_proj", "model.layers.32.mlp.experts.118.gate_proj", "model.layers.32.mlp.experts.119.gate_proj", "model.layers.32.mlp.experts.120.gate_proj", "model.layers.32.mlp.experts.121.gate_proj", "model.layers.32.mlp.experts.122.gate_proj", "model.layers.32.mlp.experts.123.gate_proj", "model.layers.32.mlp.experts.124.gate_proj", "model.layers.32.mlp.experts.125.gate_proj", "model.layers.32.mlp.experts.126.gate_proj", "model.layers.32.mlp.experts.127.gate_proj", "model.layers.32.mlp.experts.128.gate_proj", "model.layers.32.mlp.experts.129.gate_proj", "model.layers.32.mlp.experts.130.gate_proj", "model.layers.32.mlp.experts.131.gate_proj", "model.layers.32.mlp.experts.132.gate_proj", "model.layers.32.mlp.experts.133.gate_proj", "model.layers.32.mlp.experts.134.gate_proj", "model.layers.32.mlp.experts.135.gate_proj", "model.layers.32.mlp.experts.136.gate_proj", "model.layers.32.mlp.experts.137.gate_proj", "model.layers.32.mlp.experts.138.gate_proj", "model.layers.32.mlp.experts.139.gate_proj", "model.layers.32.mlp.experts.140.gate_proj", "model.layers.32.mlp.experts.141.gate_proj", "model.layers.32.mlp.experts.142.gate_proj", "model.layers.32.mlp.experts.143.gate_proj", "model.layers.32.mlp.experts.144.gate_proj", "model.layers.32.mlp.experts.145.gate_proj", "model.layers.32.mlp.experts.146.gate_proj", "model.layers.32.mlp.experts.147.gate_proj", "model.layers.32.mlp.experts.148.gate_proj", "model.layers.32.mlp.experts.149.gate_proj", "model.layers.32.mlp.experts.150.gate_proj", "model.layers.32.mlp.experts.151.gate_proj", "model.layers.32.mlp.experts.152.gate_proj", "model.layers.32.mlp.experts.153.gate_proj", "model.layers.32.mlp.experts.154.gate_proj", "model.layers.32.mlp.experts.155.gate_proj", "model.layers.32.mlp.experts.156.gate_proj", "model.layers.32.mlp.experts.157.gate_proj", "model.layers.32.mlp.experts.158.gate_proj", "model.layers.32.mlp.experts.159.gate_proj", "model.layers.32.mlp.experts.0.up_proj", "model.layers.32.mlp.experts.1.up_proj", "model.layers.32.mlp.experts.2.up_proj", "model.layers.32.mlp.experts.3.up_proj", "model.layers.32.mlp.experts.4.up_proj", "model.layers.32.mlp.experts.5.up_proj", "model.layers.32.mlp.experts.6.up_proj", "model.layers.32.mlp.experts.7.up_proj", "model.layers.32.mlp.experts.8.up_proj", "model.layers.32.mlp.experts.9.up_proj", "model.layers.32.mlp.experts.10.up_proj", "model.layers.32.mlp.experts.11.up_proj", "model.layers.32.mlp.experts.12.up_proj", "model.layers.32.mlp.experts.13.up_proj", "model.layers.32.mlp.experts.14.up_proj", "model.layers.32.mlp.experts.15.up_proj", "model.layers.32.mlp.experts.16.up_proj", "model.layers.32.mlp.experts.17.up_proj", "model.layers.32.mlp.experts.18.up_proj", "model.layers.32.mlp.experts.19.up_proj", "model.layers.32.mlp.experts.20.up_proj", "model.layers.32.mlp.experts.21.up_proj", "model.layers.32.mlp.experts.22.up_proj", "model.layers.32.mlp.experts.23.up_proj", "model.layers.32.mlp.experts.24.up_proj", "model.layers.32.mlp.experts.25.up_proj", "model.layers.32.mlp.experts.26.up_proj", "model.layers.32.mlp.experts.27.up_proj", "model.layers.32.mlp.experts.28.up_proj", "model.layers.32.mlp.experts.29.up_proj", "model.layers.32.mlp.experts.30.up_proj", "model.layers.32.mlp.experts.31.up_proj", "model.layers.32.mlp.experts.32.up_proj", "model.layers.32.mlp.experts.33.up_proj", "model.layers.32.mlp.experts.34.up_proj", "model.layers.32.mlp.experts.35.up_proj", "model.layers.32.mlp.experts.36.up_proj", "model.layers.32.mlp.experts.37.up_proj", "model.layers.32.mlp.experts.38.up_proj", "model.layers.32.mlp.experts.39.up_proj", "model.layers.32.mlp.experts.40.up_proj", "model.layers.32.mlp.experts.41.up_proj", "model.layers.32.mlp.experts.42.up_proj", "model.layers.32.mlp.experts.43.up_proj", "model.layers.32.mlp.experts.44.up_proj", "model.layers.32.mlp.experts.45.up_proj", "model.layers.32.mlp.experts.46.up_proj", "model.layers.32.mlp.experts.47.up_proj", "model.layers.32.mlp.experts.48.up_proj", "model.layers.32.mlp.experts.49.up_proj", "model.layers.32.mlp.experts.50.up_proj", "model.layers.32.mlp.experts.51.up_proj", "model.layers.32.mlp.experts.52.up_proj", "model.layers.32.mlp.experts.53.up_proj", "model.layers.32.mlp.experts.54.up_proj", "model.layers.32.mlp.experts.55.up_proj", "model.layers.32.mlp.experts.56.up_proj", "model.layers.32.mlp.experts.57.up_proj", "model.layers.32.mlp.experts.58.up_proj", "model.layers.32.mlp.experts.59.up_proj", "model.layers.32.mlp.experts.60.up_proj", "model.layers.32.mlp.experts.61.up_proj", "model.layers.32.mlp.experts.62.up_proj", "model.layers.32.mlp.experts.63.up_proj", "model.layers.32.mlp.experts.64.up_proj", "model.layers.32.mlp.experts.65.up_proj", "model.layers.32.mlp.experts.66.up_proj", "model.layers.32.mlp.experts.67.up_proj", "model.layers.32.mlp.experts.68.up_proj", "model.layers.32.mlp.experts.69.up_proj", "model.layers.32.mlp.experts.70.up_proj", "model.layers.32.mlp.experts.71.up_proj", "model.layers.32.mlp.experts.72.up_proj", "model.layers.32.mlp.experts.73.up_proj", "model.layers.32.mlp.experts.74.up_proj", "model.layers.32.mlp.experts.75.up_proj", "model.layers.32.mlp.experts.76.up_proj", "model.layers.32.mlp.experts.77.up_proj", "model.layers.32.mlp.experts.78.up_proj", "model.layers.32.mlp.experts.79.up_proj", "model.layers.32.mlp.experts.80.up_proj", "model.layers.32.mlp.experts.81.up_proj", "model.layers.32.mlp.experts.82.up_proj", "model.layers.32.mlp.experts.83.up_proj", "model.layers.32.mlp.experts.84.up_proj", "model.layers.32.mlp.experts.85.up_proj", "model.layers.32.mlp.experts.86.up_proj", "model.layers.32.mlp.experts.87.up_proj", "model.layers.32.mlp.experts.88.up_proj", "model.layers.32.mlp.experts.89.up_proj", "model.layers.32.mlp.experts.90.up_proj", "model.layers.32.mlp.experts.91.up_proj", "model.layers.32.mlp.experts.92.up_proj", "model.layers.32.mlp.experts.93.up_proj", "model.layers.32.mlp.experts.94.up_proj", "model.layers.32.mlp.experts.95.up_proj", "model.layers.32.mlp.experts.96.up_proj", "model.layers.32.mlp.experts.97.up_proj", "model.layers.32.mlp.experts.98.up_proj", "model.layers.32.mlp.experts.99.up_proj", "model.layers.32.mlp.experts.100.up_proj", "model.layers.32.mlp.experts.101.up_proj", "model.layers.32.mlp.experts.102.up_proj", "model.layers.32.mlp.experts.103.up_proj", "model.layers.32.mlp.experts.104.up_proj", "model.layers.32.mlp.experts.105.up_proj", "model.layers.32.mlp.experts.106.up_proj", "model.layers.32.mlp.experts.107.up_proj", "model.layers.32.mlp.experts.108.up_proj", "model.layers.32.mlp.experts.109.up_proj", "model.layers.32.mlp.experts.110.up_proj", "model.layers.32.mlp.experts.111.up_proj", "model.layers.32.mlp.experts.112.up_proj", "model.layers.32.mlp.experts.113.up_proj", "model.layers.32.mlp.experts.114.up_proj", "model.layers.32.mlp.experts.115.up_proj", "model.layers.32.mlp.experts.116.up_proj", "model.layers.32.mlp.experts.117.up_proj", "model.layers.32.mlp.experts.118.up_proj", "model.layers.32.mlp.experts.119.up_proj", "model.layers.32.mlp.experts.120.up_proj", "model.layers.32.mlp.experts.121.up_proj", "model.layers.32.mlp.experts.122.up_proj", "model.layers.32.mlp.experts.123.up_proj", "model.layers.32.mlp.experts.124.up_proj", "model.layers.32.mlp.experts.125.up_proj", "model.layers.32.mlp.experts.126.up_proj", "model.layers.32.mlp.experts.127.up_proj", "model.layers.32.mlp.experts.128.up_proj", "model.layers.32.mlp.experts.129.up_proj", "model.layers.32.mlp.experts.130.up_proj", "model.layers.32.mlp.experts.131.up_proj", "model.layers.32.mlp.experts.132.up_proj", "model.layers.32.mlp.experts.133.up_proj", "model.layers.32.mlp.experts.134.up_proj", "model.layers.32.mlp.experts.135.up_proj", "model.layers.32.mlp.experts.136.up_proj", "model.layers.32.mlp.experts.137.up_proj", "model.layers.32.mlp.experts.138.up_proj", "model.layers.32.mlp.experts.139.up_proj", "model.layers.32.mlp.experts.140.up_proj", "model.layers.32.mlp.experts.141.up_proj", "model.layers.32.mlp.experts.142.up_proj", "model.layers.32.mlp.experts.143.up_proj", "model.layers.32.mlp.experts.144.up_proj", "model.layers.32.mlp.experts.145.up_proj", "model.layers.32.mlp.experts.146.up_proj", "model.layers.32.mlp.experts.147.up_proj", "model.layers.32.mlp.experts.148.up_proj", "model.layers.32.mlp.experts.149.up_proj", "model.layers.32.mlp.experts.150.up_proj", "model.layers.32.mlp.experts.151.up_proj", "model.layers.32.mlp.experts.152.up_proj", "model.layers.32.mlp.experts.153.up_proj", "model.layers.32.mlp.experts.154.up_proj", "model.layers.32.mlp.experts.155.up_proj", "model.layers.32.mlp.experts.156.up_proj", "model.layers.32.mlp.experts.157.up_proj", "model.layers.32.mlp.experts.158.up_proj", "model.layers.32.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -2.041598781943343e-06, "dbits": 5033164800 } ] }, { "idx": 194, "layers": [ "model.layers.32.mlp.experts.0.down_proj", "model.layers.32.mlp.experts.1.down_proj", "model.layers.32.mlp.experts.2.down_proj", "model.layers.32.mlp.experts.3.down_proj", "model.layers.32.mlp.experts.4.down_proj", "model.layers.32.mlp.experts.5.down_proj", "model.layers.32.mlp.experts.6.down_proj", "model.layers.32.mlp.experts.7.down_proj", "model.layers.32.mlp.experts.8.down_proj", "model.layers.32.mlp.experts.9.down_proj", "model.layers.32.mlp.experts.10.down_proj", "model.layers.32.mlp.experts.11.down_proj", "model.layers.32.mlp.experts.12.down_proj", "model.layers.32.mlp.experts.13.down_proj", "model.layers.32.mlp.experts.14.down_proj", "model.layers.32.mlp.experts.15.down_proj", "model.layers.32.mlp.experts.16.down_proj", "model.layers.32.mlp.experts.17.down_proj", "model.layers.32.mlp.experts.18.down_proj", "model.layers.32.mlp.experts.19.down_proj", "model.layers.32.mlp.experts.20.down_proj", "model.layers.32.mlp.experts.21.down_proj", "model.layers.32.mlp.experts.22.down_proj", "model.layers.32.mlp.experts.23.down_proj", "model.layers.32.mlp.experts.24.down_proj", "model.layers.32.mlp.experts.25.down_proj", "model.layers.32.mlp.experts.26.down_proj", "model.layers.32.mlp.experts.27.down_proj", "model.layers.32.mlp.experts.28.down_proj", "model.layers.32.mlp.experts.29.down_proj", "model.layers.32.mlp.experts.30.down_proj", "model.layers.32.mlp.experts.31.down_proj", "model.layers.32.mlp.experts.32.down_proj", "model.layers.32.mlp.experts.33.down_proj", "model.layers.32.mlp.experts.34.down_proj", "model.layers.32.mlp.experts.35.down_proj", "model.layers.32.mlp.experts.36.down_proj", "model.layers.32.mlp.experts.37.down_proj", "model.layers.32.mlp.experts.38.down_proj", "model.layers.32.mlp.experts.39.down_proj", "model.layers.32.mlp.experts.40.down_proj", "model.layers.32.mlp.experts.41.down_proj", "model.layers.32.mlp.experts.42.down_proj", "model.layers.32.mlp.experts.43.down_proj", "model.layers.32.mlp.experts.44.down_proj", "model.layers.32.mlp.experts.45.down_proj", "model.layers.32.mlp.experts.46.down_proj", "model.layers.32.mlp.experts.47.down_proj", "model.layers.32.mlp.experts.48.down_proj", "model.layers.32.mlp.experts.49.down_proj", "model.layers.32.mlp.experts.50.down_proj", "model.layers.32.mlp.experts.51.down_proj", "model.layers.32.mlp.experts.52.down_proj", "model.layers.32.mlp.experts.53.down_proj", "model.layers.32.mlp.experts.54.down_proj", "model.layers.32.mlp.experts.55.down_proj", "model.layers.32.mlp.experts.56.down_proj", "model.layers.32.mlp.experts.57.down_proj", "model.layers.32.mlp.experts.58.down_proj", "model.layers.32.mlp.experts.59.down_proj", "model.layers.32.mlp.experts.60.down_proj", "model.layers.32.mlp.experts.61.down_proj", "model.layers.32.mlp.experts.62.down_proj", "model.layers.32.mlp.experts.63.down_proj", "model.layers.32.mlp.experts.64.down_proj", "model.layers.32.mlp.experts.65.down_proj", "model.layers.32.mlp.experts.66.down_proj", "model.layers.32.mlp.experts.67.down_proj", "model.layers.32.mlp.experts.68.down_proj", "model.layers.32.mlp.experts.69.down_proj", "model.layers.32.mlp.experts.70.down_proj", "model.layers.32.mlp.experts.71.down_proj", "model.layers.32.mlp.experts.72.down_proj", "model.layers.32.mlp.experts.73.down_proj", "model.layers.32.mlp.experts.74.down_proj", "model.layers.32.mlp.experts.75.down_proj", "model.layers.32.mlp.experts.76.down_proj", "model.layers.32.mlp.experts.77.down_proj", "model.layers.32.mlp.experts.78.down_proj", "model.layers.32.mlp.experts.79.down_proj", "model.layers.32.mlp.experts.80.down_proj", "model.layers.32.mlp.experts.81.down_proj", "model.layers.32.mlp.experts.82.down_proj", "model.layers.32.mlp.experts.83.down_proj", "model.layers.32.mlp.experts.84.down_proj", "model.layers.32.mlp.experts.85.down_proj", "model.layers.32.mlp.experts.86.down_proj", "model.layers.32.mlp.experts.87.down_proj", "model.layers.32.mlp.experts.88.down_proj", "model.layers.32.mlp.experts.89.down_proj", "model.layers.32.mlp.experts.90.down_proj", "model.layers.32.mlp.experts.91.down_proj", "model.layers.32.mlp.experts.92.down_proj", "model.layers.32.mlp.experts.93.down_proj", "model.layers.32.mlp.experts.94.down_proj", "model.layers.32.mlp.experts.95.down_proj", "model.layers.32.mlp.experts.96.down_proj", "model.layers.32.mlp.experts.97.down_proj", "model.layers.32.mlp.experts.98.down_proj", "model.layers.32.mlp.experts.99.down_proj", "model.layers.32.mlp.experts.100.down_proj", "model.layers.32.mlp.experts.101.down_proj", "model.layers.32.mlp.experts.102.down_proj", "model.layers.32.mlp.experts.103.down_proj", "model.layers.32.mlp.experts.104.down_proj", "model.layers.32.mlp.experts.105.down_proj", "model.layers.32.mlp.experts.106.down_proj", "model.layers.32.mlp.experts.107.down_proj", "model.layers.32.mlp.experts.108.down_proj", "model.layers.32.mlp.experts.109.down_proj", "model.layers.32.mlp.experts.110.down_proj", "model.layers.32.mlp.experts.111.down_proj", "model.layers.32.mlp.experts.112.down_proj", "model.layers.32.mlp.experts.113.down_proj", "model.layers.32.mlp.experts.114.down_proj", "model.layers.32.mlp.experts.115.down_proj", "model.layers.32.mlp.experts.116.down_proj", "model.layers.32.mlp.experts.117.down_proj", "model.layers.32.mlp.experts.118.down_proj", "model.layers.32.mlp.experts.119.down_proj", "model.layers.32.mlp.experts.120.down_proj", "model.layers.32.mlp.experts.121.down_proj", "model.layers.32.mlp.experts.122.down_proj", "model.layers.32.mlp.experts.123.down_proj", "model.layers.32.mlp.experts.124.down_proj", "model.layers.32.mlp.experts.125.down_proj", "model.layers.32.mlp.experts.126.down_proj", "model.layers.32.mlp.experts.127.down_proj", "model.layers.32.mlp.experts.128.down_proj", "model.layers.32.mlp.experts.129.down_proj", "model.layers.32.mlp.experts.130.down_proj", "model.layers.32.mlp.experts.131.down_proj", "model.layers.32.mlp.experts.132.down_proj", "model.layers.32.mlp.experts.133.down_proj", "model.layers.32.mlp.experts.134.down_proj", "model.layers.32.mlp.experts.135.down_proj", "model.layers.32.mlp.experts.136.down_proj", "model.layers.32.mlp.experts.137.down_proj", "model.layers.32.mlp.experts.138.down_proj", "model.layers.32.mlp.experts.139.down_proj", "model.layers.32.mlp.experts.140.down_proj", "model.layers.32.mlp.experts.141.down_proj", "model.layers.32.mlp.experts.142.down_proj", "model.layers.32.mlp.experts.143.down_proj", "model.layers.32.mlp.experts.144.down_proj", "model.layers.32.mlp.experts.145.down_proj", "model.layers.32.mlp.experts.146.down_proj", "model.layers.32.mlp.experts.147.down_proj", "model.layers.32.mlp.experts.148.down_proj", "model.layers.32.mlp.experts.149.down_proj", "model.layers.32.mlp.experts.150.down_proj", "model.layers.32.mlp.experts.151.down_proj", "model.layers.32.mlp.experts.152.down_proj", "model.layers.32.mlp.experts.153.down_proj", "model.layers.32.mlp.experts.154.down_proj", "model.layers.32.mlp.experts.155.down_proj", "model.layers.32.mlp.experts.156.down_proj", "model.layers.32.mlp.experts.157.down_proj", "model.layers.32.mlp.experts.158.down_proj", "model.layers.32.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -9.661045623943643e-07, "dbits": 2516582400 } ] }, { "idx": 195, "layers": [ "model.layers.33.self_attn.q_proj" ], "candidates": [ { "dkld": 6.0004065744578405e-06, "dbits": 125829120 } ] }, { "idx": 196, "layers": [ "model.layers.33.self_attn.k_proj", "model.layers.33.self_attn.v_proj" ], "candidates": [ { "dkld": -3.3173244446515603e-06, "dbits": 20971520 } ] }, { "idx": 197, "layers": [ "model.layers.33.self_attn.o_proj" ], "candidates": [ { "dkld": 5.825798143632667e-06, "dbits": 125829120 } ] }, { "idx": 198, "layers": [ "model.layers.33.mlp.shared_experts.gate_proj", "model.layers.33.mlp.shared_experts.up_proj", "model.layers.33.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.7903617592528896e-06, "dbits": 47185920 } ] }, { "idx": 199, "layers": [ "model.layers.33.mlp.experts.0.gate_proj", "model.layers.33.mlp.experts.1.gate_proj", "model.layers.33.mlp.experts.2.gate_proj", "model.layers.33.mlp.experts.3.gate_proj", "model.layers.33.mlp.experts.4.gate_proj", "model.layers.33.mlp.experts.5.gate_proj", "model.layers.33.mlp.experts.6.gate_proj", "model.layers.33.mlp.experts.7.gate_proj", "model.layers.33.mlp.experts.8.gate_proj", "model.layers.33.mlp.experts.9.gate_proj", "model.layers.33.mlp.experts.10.gate_proj", "model.layers.33.mlp.experts.11.gate_proj", "model.layers.33.mlp.experts.12.gate_proj", "model.layers.33.mlp.experts.13.gate_proj", "model.layers.33.mlp.experts.14.gate_proj", "model.layers.33.mlp.experts.15.gate_proj", "model.layers.33.mlp.experts.16.gate_proj", "model.layers.33.mlp.experts.17.gate_proj", "model.layers.33.mlp.experts.18.gate_proj", "model.layers.33.mlp.experts.19.gate_proj", "model.layers.33.mlp.experts.20.gate_proj", "model.layers.33.mlp.experts.21.gate_proj", "model.layers.33.mlp.experts.22.gate_proj", "model.layers.33.mlp.experts.23.gate_proj", "model.layers.33.mlp.experts.24.gate_proj", "model.layers.33.mlp.experts.25.gate_proj", "model.layers.33.mlp.experts.26.gate_proj", "model.layers.33.mlp.experts.27.gate_proj", "model.layers.33.mlp.experts.28.gate_proj", "model.layers.33.mlp.experts.29.gate_proj", "model.layers.33.mlp.experts.30.gate_proj", "model.layers.33.mlp.experts.31.gate_proj", "model.layers.33.mlp.experts.32.gate_proj", "model.layers.33.mlp.experts.33.gate_proj", "model.layers.33.mlp.experts.34.gate_proj", "model.layers.33.mlp.experts.35.gate_proj", "model.layers.33.mlp.experts.36.gate_proj", "model.layers.33.mlp.experts.37.gate_proj", "model.layers.33.mlp.experts.38.gate_proj", "model.layers.33.mlp.experts.39.gate_proj", "model.layers.33.mlp.experts.40.gate_proj", "model.layers.33.mlp.experts.41.gate_proj", "model.layers.33.mlp.experts.42.gate_proj", "model.layers.33.mlp.experts.43.gate_proj", "model.layers.33.mlp.experts.44.gate_proj", "model.layers.33.mlp.experts.45.gate_proj", "model.layers.33.mlp.experts.46.gate_proj", "model.layers.33.mlp.experts.47.gate_proj", "model.layers.33.mlp.experts.48.gate_proj", "model.layers.33.mlp.experts.49.gate_proj", "model.layers.33.mlp.experts.50.gate_proj", "model.layers.33.mlp.experts.51.gate_proj", "model.layers.33.mlp.experts.52.gate_proj", "model.layers.33.mlp.experts.53.gate_proj", "model.layers.33.mlp.experts.54.gate_proj", "model.layers.33.mlp.experts.55.gate_proj", "model.layers.33.mlp.experts.56.gate_proj", "model.layers.33.mlp.experts.57.gate_proj", "model.layers.33.mlp.experts.58.gate_proj", "model.layers.33.mlp.experts.59.gate_proj", "model.layers.33.mlp.experts.60.gate_proj", "model.layers.33.mlp.experts.61.gate_proj", "model.layers.33.mlp.experts.62.gate_proj", "model.layers.33.mlp.experts.63.gate_proj", "model.layers.33.mlp.experts.64.gate_proj", "model.layers.33.mlp.experts.65.gate_proj", "model.layers.33.mlp.experts.66.gate_proj", "model.layers.33.mlp.experts.67.gate_proj", "model.layers.33.mlp.experts.68.gate_proj", "model.layers.33.mlp.experts.69.gate_proj", "model.layers.33.mlp.experts.70.gate_proj", "model.layers.33.mlp.experts.71.gate_proj", "model.layers.33.mlp.experts.72.gate_proj", "model.layers.33.mlp.experts.73.gate_proj", "model.layers.33.mlp.experts.74.gate_proj", "model.layers.33.mlp.experts.75.gate_proj", "model.layers.33.mlp.experts.76.gate_proj", "model.layers.33.mlp.experts.77.gate_proj", "model.layers.33.mlp.experts.78.gate_proj", "model.layers.33.mlp.experts.79.gate_proj", "model.layers.33.mlp.experts.80.gate_proj", "model.layers.33.mlp.experts.81.gate_proj", "model.layers.33.mlp.experts.82.gate_proj", "model.layers.33.mlp.experts.83.gate_proj", "model.layers.33.mlp.experts.84.gate_proj", "model.layers.33.mlp.experts.85.gate_proj", "model.layers.33.mlp.experts.86.gate_proj", "model.layers.33.mlp.experts.87.gate_proj", "model.layers.33.mlp.experts.88.gate_proj", "model.layers.33.mlp.experts.89.gate_proj", "model.layers.33.mlp.experts.90.gate_proj", "model.layers.33.mlp.experts.91.gate_proj", "model.layers.33.mlp.experts.92.gate_proj", "model.layers.33.mlp.experts.93.gate_proj", "model.layers.33.mlp.experts.94.gate_proj", "model.layers.33.mlp.experts.95.gate_proj", "model.layers.33.mlp.experts.96.gate_proj", "model.layers.33.mlp.experts.97.gate_proj", "model.layers.33.mlp.experts.98.gate_proj", "model.layers.33.mlp.experts.99.gate_proj", "model.layers.33.mlp.experts.100.gate_proj", "model.layers.33.mlp.experts.101.gate_proj", "model.layers.33.mlp.experts.102.gate_proj", "model.layers.33.mlp.experts.103.gate_proj", "model.layers.33.mlp.experts.104.gate_proj", "model.layers.33.mlp.experts.105.gate_proj", "model.layers.33.mlp.experts.106.gate_proj", "model.layers.33.mlp.experts.107.gate_proj", "model.layers.33.mlp.experts.108.gate_proj", "model.layers.33.mlp.experts.109.gate_proj", "model.layers.33.mlp.experts.110.gate_proj", "model.layers.33.mlp.experts.111.gate_proj", "model.layers.33.mlp.experts.112.gate_proj", "model.layers.33.mlp.experts.113.gate_proj", "model.layers.33.mlp.experts.114.gate_proj", "model.layers.33.mlp.experts.115.gate_proj", "model.layers.33.mlp.experts.116.gate_proj", "model.layers.33.mlp.experts.117.gate_proj", "model.layers.33.mlp.experts.118.gate_proj", "model.layers.33.mlp.experts.119.gate_proj", "model.layers.33.mlp.experts.120.gate_proj", "model.layers.33.mlp.experts.121.gate_proj", "model.layers.33.mlp.experts.122.gate_proj", "model.layers.33.mlp.experts.123.gate_proj", "model.layers.33.mlp.experts.124.gate_proj", "model.layers.33.mlp.experts.125.gate_proj", "model.layers.33.mlp.experts.126.gate_proj", "model.layers.33.mlp.experts.127.gate_proj", "model.layers.33.mlp.experts.128.gate_proj", "model.layers.33.mlp.experts.129.gate_proj", "model.layers.33.mlp.experts.130.gate_proj", "model.layers.33.mlp.experts.131.gate_proj", "model.layers.33.mlp.experts.132.gate_proj", "model.layers.33.mlp.experts.133.gate_proj", "model.layers.33.mlp.experts.134.gate_proj", "model.layers.33.mlp.experts.135.gate_proj", "model.layers.33.mlp.experts.136.gate_proj", "model.layers.33.mlp.experts.137.gate_proj", "model.layers.33.mlp.experts.138.gate_proj", "model.layers.33.mlp.experts.139.gate_proj", "model.layers.33.mlp.experts.140.gate_proj", "model.layers.33.mlp.experts.141.gate_proj", "model.layers.33.mlp.experts.142.gate_proj", "model.layers.33.mlp.experts.143.gate_proj", "model.layers.33.mlp.experts.144.gate_proj", "model.layers.33.mlp.experts.145.gate_proj", "model.layers.33.mlp.experts.146.gate_proj", "model.layers.33.mlp.experts.147.gate_proj", "model.layers.33.mlp.experts.148.gate_proj", "model.layers.33.mlp.experts.149.gate_proj", "model.layers.33.mlp.experts.150.gate_proj", "model.layers.33.mlp.experts.151.gate_proj", "model.layers.33.mlp.experts.152.gate_proj", "model.layers.33.mlp.experts.153.gate_proj", "model.layers.33.mlp.experts.154.gate_proj", "model.layers.33.mlp.experts.155.gate_proj", "model.layers.33.mlp.experts.156.gate_proj", "model.layers.33.mlp.experts.157.gate_proj", "model.layers.33.mlp.experts.158.gate_proj", "model.layers.33.mlp.experts.159.gate_proj", "model.layers.33.mlp.experts.0.up_proj", "model.layers.33.mlp.experts.1.up_proj", "model.layers.33.mlp.experts.2.up_proj", "model.layers.33.mlp.experts.3.up_proj", "model.layers.33.mlp.experts.4.up_proj", "model.layers.33.mlp.experts.5.up_proj", "model.layers.33.mlp.experts.6.up_proj", "model.layers.33.mlp.experts.7.up_proj", "model.layers.33.mlp.experts.8.up_proj", "model.layers.33.mlp.experts.9.up_proj", "model.layers.33.mlp.experts.10.up_proj", "model.layers.33.mlp.experts.11.up_proj", "model.layers.33.mlp.experts.12.up_proj", "model.layers.33.mlp.experts.13.up_proj", "model.layers.33.mlp.experts.14.up_proj", "model.layers.33.mlp.experts.15.up_proj", "model.layers.33.mlp.experts.16.up_proj", "model.layers.33.mlp.experts.17.up_proj", "model.layers.33.mlp.experts.18.up_proj", "model.layers.33.mlp.experts.19.up_proj", "model.layers.33.mlp.experts.20.up_proj", "model.layers.33.mlp.experts.21.up_proj", "model.layers.33.mlp.experts.22.up_proj", "model.layers.33.mlp.experts.23.up_proj", "model.layers.33.mlp.experts.24.up_proj", "model.layers.33.mlp.experts.25.up_proj", "model.layers.33.mlp.experts.26.up_proj", "model.layers.33.mlp.experts.27.up_proj", "model.layers.33.mlp.experts.28.up_proj", "model.layers.33.mlp.experts.29.up_proj", "model.layers.33.mlp.experts.30.up_proj", "model.layers.33.mlp.experts.31.up_proj", "model.layers.33.mlp.experts.32.up_proj", "model.layers.33.mlp.experts.33.up_proj", "model.layers.33.mlp.experts.34.up_proj", "model.layers.33.mlp.experts.35.up_proj", "model.layers.33.mlp.experts.36.up_proj", "model.layers.33.mlp.experts.37.up_proj", "model.layers.33.mlp.experts.38.up_proj", "model.layers.33.mlp.experts.39.up_proj", "model.layers.33.mlp.experts.40.up_proj", "model.layers.33.mlp.experts.41.up_proj", "model.layers.33.mlp.experts.42.up_proj", "model.layers.33.mlp.experts.43.up_proj", "model.layers.33.mlp.experts.44.up_proj", "model.layers.33.mlp.experts.45.up_proj", "model.layers.33.mlp.experts.46.up_proj", "model.layers.33.mlp.experts.47.up_proj", "model.layers.33.mlp.experts.48.up_proj", "model.layers.33.mlp.experts.49.up_proj", "model.layers.33.mlp.experts.50.up_proj", "model.layers.33.mlp.experts.51.up_proj", "model.layers.33.mlp.experts.52.up_proj", "model.layers.33.mlp.experts.53.up_proj", "model.layers.33.mlp.experts.54.up_proj", "model.layers.33.mlp.experts.55.up_proj", "model.layers.33.mlp.experts.56.up_proj", "model.layers.33.mlp.experts.57.up_proj", "model.layers.33.mlp.experts.58.up_proj", "model.layers.33.mlp.experts.59.up_proj", "model.layers.33.mlp.experts.60.up_proj", "model.layers.33.mlp.experts.61.up_proj", "model.layers.33.mlp.experts.62.up_proj", "model.layers.33.mlp.experts.63.up_proj", "model.layers.33.mlp.experts.64.up_proj", "model.layers.33.mlp.experts.65.up_proj", "model.layers.33.mlp.experts.66.up_proj", "model.layers.33.mlp.experts.67.up_proj", "model.layers.33.mlp.experts.68.up_proj", "model.layers.33.mlp.experts.69.up_proj", "model.layers.33.mlp.experts.70.up_proj", "model.layers.33.mlp.experts.71.up_proj", "model.layers.33.mlp.experts.72.up_proj", "model.layers.33.mlp.experts.73.up_proj", "model.layers.33.mlp.experts.74.up_proj", "model.layers.33.mlp.experts.75.up_proj", "model.layers.33.mlp.experts.76.up_proj", "model.layers.33.mlp.experts.77.up_proj", "model.layers.33.mlp.experts.78.up_proj", "model.layers.33.mlp.experts.79.up_proj", "model.layers.33.mlp.experts.80.up_proj", "model.layers.33.mlp.experts.81.up_proj", "model.layers.33.mlp.experts.82.up_proj", "model.layers.33.mlp.experts.83.up_proj", "model.layers.33.mlp.experts.84.up_proj", "model.layers.33.mlp.experts.85.up_proj", "model.layers.33.mlp.experts.86.up_proj", "model.layers.33.mlp.experts.87.up_proj", "model.layers.33.mlp.experts.88.up_proj", "model.layers.33.mlp.experts.89.up_proj", "model.layers.33.mlp.experts.90.up_proj", "model.layers.33.mlp.experts.91.up_proj", "model.layers.33.mlp.experts.92.up_proj", "model.layers.33.mlp.experts.93.up_proj", "model.layers.33.mlp.experts.94.up_proj", "model.layers.33.mlp.experts.95.up_proj", "model.layers.33.mlp.experts.96.up_proj", "model.layers.33.mlp.experts.97.up_proj", "model.layers.33.mlp.experts.98.up_proj", "model.layers.33.mlp.experts.99.up_proj", "model.layers.33.mlp.experts.100.up_proj", "model.layers.33.mlp.experts.101.up_proj", "model.layers.33.mlp.experts.102.up_proj", "model.layers.33.mlp.experts.103.up_proj", "model.layers.33.mlp.experts.104.up_proj", "model.layers.33.mlp.experts.105.up_proj", "model.layers.33.mlp.experts.106.up_proj", "model.layers.33.mlp.experts.107.up_proj", "model.layers.33.mlp.experts.108.up_proj", "model.layers.33.mlp.experts.109.up_proj", "model.layers.33.mlp.experts.110.up_proj", "model.layers.33.mlp.experts.111.up_proj", "model.layers.33.mlp.experts.112.up_proj", "model.layers.33.mlp.experts.113.up_proj", "model.layers.33.mlp.experts.114.up_proj", "model.layers.33.mlp.experts.115.up_proj", "model.layers.33.mlp.experts.116.up_proj", "model.layers.33.mlp.experts.117.up_proj", "model.layers.33.mlp.experts.118.up_proj", "model.layers.33.mlp.experts.119.up_proj", "model.layers.33.mlp.experts.120.up_proj", "model.layers.33.mlp.experts.121.up_proj", "model.layers.33.mlp.experts.122.up_proj", "model.layers.33.mlp.experts.123.up_proj", "model.layers.33.mlp.experts.124.up_proj", "model.layers.33.mlp.experts.125.up_proj", "model.layers.33.mlp.experts.126.up_proj", "model.layers.33.mlp.experts.127.up_proj", "model.layers.33.mlp.experts.128.up_proj", "model.layers.33.mlp.experts.129.up_proj", "model.layers.33.mlp.experts.130.up_proj", "model.layers.33.mlp.experts.131.up_proj", "model.layers.33.mlp.experts.132.up_proj", "model.layers.33.mlp.experts.133.up_proj", "model.layers.33.mlp.experts.134.up_proj", "model.layers.33.mlp.experts.135.up_proj", "model.layers.33.mlp.experts.136.up_proj", "model.layers.33.mlp.experts.137.up_proj", "model.layers.33.mlp.experts.138.up_proj", "model.layers.33.mlp.experts.139.up_proj", "model.layers.33.mlp.experts.140.up_proj", "model.layers.33.mlp.experts.141.up_proj", "model.layers.33.mlp.experts.142.up_proj", "model.layers.33.mlp.experts.143.up_proj", "model.layers.33.mlp.experts.144.up_proj", "model.layers.33.mlp.experts.145.up_proj", "model.layers.33.mlp.experts.146.up_proj", "model.layers.33.mlp.experts.147.up_proj", "model.layers.33.mlp.experts.148.up_proj", "model.layers.33.mlp.experts.149.up_proj", "model.layers.33.mlp.experts.150.up_proj", "model.layers.33.mlp.experts.151.up_proj", "model.layers.33.mlp.experts.152.up_proj", "model.layers.33.mlp.experts.153.up_proj", "model.layers.33.mlp.experts.154.up_proj", "model.layers.33.mlp.experts.155.up_proj", "model.layers.33.mlp.experts.156.up_proj", "model.layers.33.mlp.experts.157.up_proj", "model.layers.33.mlp.experts.158.up_proj", "model.layers.33.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -2.728617982938929e-06, "dbits": 5033164800 } ] }, { "idx": 200, "layers": [ "model.layers.33.mlp.experts.0.down_proj", "model.layers.33.mlp.experts.1.down_proj", "model.layers.33.mlp.experts.2.down_proj", "model.layers.33.mlp.experts.3.down_proj", "model.layers.33.mlp.experts.4.down_proj", "model.layers.33.mlp.experts.5.down_proj", "model.layers.33.mlp.experts.6.down_proj", "model.layers.33.mlp.experts.7.down_proj", "model.layers.33.mlp.experts.8.down_proj", "model.layers.33.mlp.experts.9.down_proj", "model.layers.33.mlp.experts.10.down_proj", "model.layers.33.mlp.experts.11.down_proj", "model.layers.33.mlp.experts.12.down_proj", "model.layers.33.mlp.experts.13.down_proj", "model.layers.33.mlp.experts.14.down_proj", "model.layers.33.mlp.experts.15.down_proj", "model.layers.33.mlp.experts.16.down_proj", "model.layers.33.mlp.experts.17.down_proj", "model.layers.33.mlp.experts.18.down_proj", "model.layers.33.mlp.experts.19.down_proj", "model.layers.33.mlp.experts.20.down_proj", "model.layers.33.mlp.experts.21.down_proj", "model.layers.33.mlp.experts.22.down_proj", "model.layers.33.mlp.experts.23.down_proj", "model.layers.33.mlp.experts.24.down_proj", "model.layers.33.mlp.experts.25.down_proj", "model.layers.33.mlp.experts.26.down_proj", "model.layers.33.mlp.experts.27.down_proj", "model.layers.33.mlp.experts.28.down_proj", "model.layers.33.mlp.experts.29.down_proj", "model.layers.33.mlp.experts.30.down_proj", "model.layers.33.mlp.experts.31.down_proj", "model.layers.33.mlp.experts.32.down_proj", "model.layers.33.mlp.experts.33.down_proj", "model.layers.33.mlp.experts.34.down_proj", "model.layers.33.mlp.experts.35.down_proj", "model.layers.33.mlp.experts.36.down_proj", "model.layers.33.mlp.experts.37.down_proj", "model.layers.33.mlp.experts.38.down_proj", "model.layers.33.mlp.experts.39.down_proj", "model.layers.33.mlp.experts.40.down_proj", "model.layers.33.mlp.experts.41.down_proj", "model.layers.33.mlp.experts.42.down_proj", "model.layers.33.mlp.experts.43.down_proj", "model.layers.33.mlp.experts.44.down_proj", "model.layers.33.mlp.experts.45.down_proj", "model.layers.33.mlp.experts.46.down_proj", "model.layers.33.mlp.experts.47.down_proj", "model.layers.33.mlp.experts.48.down_proj", "model.layers.33.mlp.experts.49.down_proj", "model.layers.33.mlp.experts.50.down_proj", "model.layers.33.mlp.experts.51.down_proj", "model.layers.33.mlp.experts.52.down_proj", "model.layers.33.mlp.experts.53.down_proj", "model.layers.33.mlp.experts.54.down_proj", "model.layers.33.mlp.experts.55.down_proj", "model.layers.33.mlp.experts.56.down_proj", "model.layers.33.mlp.experts.57.down_proj", "model.layers.33.mlp.experts.58.down_proj", "model.layers.33.mlp.experts.59.down_proj", "model.layers.33.mlp.experts.60.down_proj", "model.layers.33.mlp.experts.61.down_proj", "model.layers.33.mlp.experts.62.down_proj", "model.layers.33.mlp.experts.63.down_proj", "model.layers.33.mlp.experts.64.down_proj", "model.layers.33.mlp.experts.65.down_proj", "model.layers.33.mlp.experts.66.down_proj", "model.layers.33.mlp.experts.67.down_proj", "model.layers.33.mlp.experts.68.down_proj", "model.layers.33.mlp.experts.69.down_proj", "model.layers.33.mlp.experts.70.down_proj", "model.layers.33.mlp.experts.71.down_proj", "model.layers.33.mlp.experts.72.down_proj", "model.layers.33.mlp.experts.73.down_proj", "model.layers.33.mlp.experts.74.down_proj", "model.layers.33.mlp.experts.75.down_proj", "model.layers.33.mlp.experts.76.down_proj", "model.layers.33.mlp.experts.77.down_proj", "model.layers.33.mlp.experts.78.down_proj", "model.layers.33.mlp.experts.79.down_proj", "model.layers.33.mlp.experts.80.down_proj", "model.layers.33.mlp.experts.81.down_proj", "model.layers.33.mlp.experts.82.down_proj", "model.layers.33.mlp.experts.83.down_proj", "model.layers.33.mlp.experts.84.down_proj", "model.layers.33.mlp.experts.85.down_proj", "model.layers.33.mlp.experts.86.down_proj", "model.layers.33.mlp.experts.87.down_proj", "model.layers.33.mlp.experts.88.down_proj", "model.layers.33.mlp.experts.89.down_proj", "model.layers.33.mlp.experts.90.down_proj", "model.layers.33.mlp.experts.91.down_proj", "model.layers.33.mlp.experts.92.down_proj", "model.layers.33.mlp.experts.93.down_proj", "model.layers.33.mlp.experts.94.down_proj", "model.layers.33.mlp.experts.95.down_proj", "model.layers.33.mlp.experts.96.down_proj", "model.layers.33.mlp.experts.97.down_proj", "model.layers.33.mlp.experts.98.down_proj", "model.layers.33.mlp.experts.99.down_proj", "model.layers.33.mlp.experts.100.down_proj", "model.layers.33.mlp.experts.101.down_proj", "model.layers.33.mlp.experts.102.down_proj", "model.layers.33.mlp.experts.103.down_proj", "model.layers.33.mlp.experts.104.down_proj", "model.layers.33.mlp.experts.105.down_proj", "model.layers.33.mlp.experts.106.down_proj", "model.layers.33.mlp.experts.107.down_proj", "model.layers.33.mlp.experts.108.down_proj", "model.layers.33.mlp.experts.109.down_proj", "model.layers.33.mlp.experts.110.down_proj", "model.layers.33.mlp.experts.111.down_proj", "model.layers.33.mlp.experts.112.down_proj", "model.layers.33.mlp.experts.113.down_proj", "model.layers.33.mlp.experts.114.down_proj", "model.layers.33.mlp.experts.115.down_proj", "model.layers.33.mlp.experts.116.down_proj", "model.layers.33.mlp.experts.117.down_proj", "model.layers.33.mlp.experts.118.down_proj", "model.layers.33.mlp.experts.119.down_proj", "model.layers.33.mlp.experts.120.down_proj", "model.layers.33.mlp.experts.121.down_proj", "model.layers.33.mlp.experts.122.down_proj", "model.layers.33.mlp.experts.123.down_proj", "model.layers.33.mlp.experts.124.down_proj", "model.layers.33.mlp.experts.125.down_proj", "model.layers.33.mlp.experts.126.down_proj", "model.layers.33.mlp.experts.127.down_proj", "model.layers.33.mlp.experts.128.down_proj", "model.layers.33.mlp.experts.129.down_proj", "model.layers.33.mlp.experts.130.down_proj", "model.layers.33.mlp.experts.131.down_proj", "model.layers.33.mlp.experts.132.down_proj", "model.layers.33.mlp.experts.133.down_proj", "model.layers.33.mlp.experts.134.down_proj", "model.layers.33.mlp.experts.135.down_proj", "model.layers.33.mlp.experts.136.down_proj", "model.layers.33.mlp.experts.137.down_proj", "model.layers.33.mlp.experts.138.down_proj", "model.layers.33.mlp.experts.139.down_proj", "model.layers.33.mlp.experts.140.down_proj", "model.layers.33.mlp.experts.141.down_proj", "model.layers.33.mlp.experts.142.down_proj", "model.layers.33.mlp.experts.143.down_proj", "model.layers.33.mlp.experts.144.down_proj", "model.layers.33.mlp.experts.145.down_proj", "model.layers.33.mlp.experts.146.down_proj", "model.layers.33.mlp.experts.147.down_proj", "model.layers.33.mlp.experts.148.down_proj", "model.layers.33.mlp.experts.149.down_proj", "model.layers.33.mlp.experts.150.down_proj", "model.layers.33.mlp.experts.151.down_proj", "model.layers.33.mlp.experts.152.down_proj", "model.layers.33.mlp.experts.153.down_proj", "model.layers.33.mlp.experts.154.down_proj", "model.layers.33.mlp.experts.155.down_proj", "model.layers.33.mlp.experts.156.down_proj", "model.layers.33.mlp.experts.157.down_proj", "model.layers.33.mlp.experts.158.down_proj", "model.layers.33.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.519240999594309e-06, "dbits": 2516582400 } ] }, { "idx": 201, "layers": [ "model.layers.34.self_attn.q_proj" ], "candidates": [ { "dkld": -1.1780473869293494e-06, "dbits": 125829120 } ] }, { "idx": 202, "layers": [ "model.layers.34.self_attn.k_proj", "model.layers.34.self_attn.v_proj" ], "candidates": [ { "dkld": -7.0927431806922176e-06, "dbits": 20971520 } ] }, { "idx": 203, "layers": [ "model.layers.34.self_attn.o_proj" ], "candidates": [ { "dkld": -1.1852249735966303e-05, "dbits": 125829120 } ] }, { "idx": 204, "layers": [ "model.layers.34.mlp.shared_experts.gate_proj", "model.layers.34.mlp.shared_experts.up_proj", "model.layers.34.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.556437488645315e-06, "dbits": 47185920 } ] }, { "idx": 205, "layers": [ "model.layers.34.mlp.experts.0.gate_proj", "model.layers.34.mlp.experts.1.gate_proj", "model.layers.34.mlp.experts.2.gate_proj", "model.layers.34.mlp.experts.3.gate_proj", "model.layers.34.mlp.experts.4.gate_proj", "model.layers.34.mlp.experts.5.gate_proj", "model.layers.34.mlp.experts.6.gate_proj", "model.layers.34.mlp.experts.7.gate_proj", "model.layers.34.mlp.experts.8.gate_proj", "model.layers.34.mlp.experts.9.gate_proj", "model.layers.34.mlp.experts.10.gate_proj", "model.layers.34.mlp.experts.11.gate_proj", "model.layers.34.mlp.experts.12.gate_proj", "model.layers.34.mlp.experts.13.gate_proj", "model.layers.34.mlp.experts.14.gate_proj", "model.layers.34.mlp.experts.15.gate_proj", "model.layers.34.mlp.experts.16.gate_proj", "model.layers.34.mlp.experts.17.gate_proj", "model.layers.34.mlp.experts.18.gate_proj", "model.layers.34.mlp.experts.19.gate_proj", "model.layers.34.mlp.experts.20.gate_proj", "model.layers.34.mlp.experts.21.gate_proj", "model.layers.34.mlp.experts.22.gate_proj", "model.layers.34.mlp.experts.23.gate_proj", "model.layers.34.mlp.experts.24.gate_proj", "model.layers.34.mlp.experts.25.gate_proj", "model.layers.34.mlp.experts.26.gate_proj", "model.layers.34.mlp.experts.27.gate_proj", "model.layers.34.mlp.experts.28.gate_proj", "model.layers.34.mlp.experts.29.gate_proj", "model.layers.34.mlp.experts.30.gate_proj", "model.layers.34.mlp.experts.31.gate_proj", "model.layers.34.mlp.experts.32.gate_proj", "model.layers.34.mlp.experts.33.gate_proj", "model.layers.34.mlp.experts.34.gate_proj", "model.layers.34.mlp.experts.35.gate_proj", "model.layers.34.mlp.experts.36.gate_proj", "model.layers.34.mlp.experts.37.gate_proj", "model.layers.34.mlp.experts.38.gate_proj", "model.layers.34.mlp.experts.39.gate_proj", "model.layers.34.mlp.experts.40.gate_proj", "model.layers.34.mlp.experts.41.gate_proj", "model.layers.34.mlp.experts.42.gate_proj", "model.layers.34.mlp.experts.43.gate_proj", "model.layers.34.mlp.experts.44.gate_proj", "model.layers.34.mlp.experts.45.gate_proj", "model.layers.34.mlp.experts.46.gate_proj", "model.layers.34.mlp.experts.47.gate_proj", "model.layers.34.mlp.experts.48.gate_proj", "model.layers.34.mlp.experts.49.gate_proj", "model.layers.34.mlp.experts.50.gate_proj", "model.layers.34.mlp.experts.51.gate_proj", "model.layers.34.mlp.experts.52.gate_proj", "model.layers.34.mlp.experts.53.gate_proj", "model.layers.34.mlp.experts.54.gate_proj", "model.layers.34.mlp.experts.55.gate_proj", "model.layers.34.mlp.experts.56.gate_proj", "model.layers.34.mlp.experts.57.gate_proj", "model.layers.34.mlp.experts.58.gate_proj", "model.layers.34.mlp.experts.59.gate_proj", "model.layers.34.mlp.experts.60.gate_proj", "model.layers.34.mlp.experts.61.gate_proj", "model.layers.34.mlp.experts.62.gate_proj", "model.layers.34.mlp.experts.63.gate_proj", "model.layers.34.mlp.experts.64.gate_proj", "model.layers.34.mlp.experts.65.gate_proj", "model.layers.34.mlp.experts.66.gate_proj", "model.layers.34.mlp.experts.67.gate_proj", "model.layers.34.mlp.experts.68.gate_proj", "model.layers.34.mlp.experts.69.gate_proj", "model.layers.34.mlp.experts.70.gate_proj", "model.layers.34.mlp.experts.71.gate_proj", "model.layers.34.mlp.experts.72.gate_proj", "model.layers.34.mlp.experts.73.gate_proj", "model.layers.34.mlp.experts.74.gate_proj", "model.layers.34.mlp.experts.75.gate_proj", "model.layers.34.mlp.experts.76.gate_proj", "model.layers.34.mlp.experts.77.gate_proj", "model.layers.34.mlp.experts.78.gate_proj", "model.layers.34.mlp.experts.79.gate_proj", "model.layers.34.mlp.experts.80.gate_proj", "model.layers.34.mlp.experts.81.gate_proj", "model.layers.34.mlp.experts.82.gate_proj", "model.layers.34.mlp.experts.83.gate_proj", "model.layers.34.mlp.experts.84.gate_proj", "model.layers.34.mlp.experts.85.gate_proj", "model.layers.34.mlp.experts.86.gate_proj", "model.layers.34.mlp.experts.87.gate_proj", "model.layers.34.mlp.experts.88.gate_proj", "model.layers.34.mlp.experts.89.gate_proj", "model.layers.34.mlp.experts.90.gate_proj", "model.layers.34.mlp.experts.91.gate_proj", "model.layers.34.mlp.experts.92.gate_proj", "model.layers.34.mlp.experts.93.gate_proj", "model.layers.34.mlp.experts.94.gate_proj", "model.layers.34.mlp.experts.95.gate_proj", "model.layers.34.mlp.experts.96.gate_proj", "model.layers.34.mlp.experts.97.gate_proj", "model.layers.34.mlp.experts.98.gate_proj", "model.layers.34.mlp.experts.99.gate_proj", "model.layers.34.mlp.experts.100.gate_proj", "model.layers.34.mlp.experts.101.gate_proj", "model.layers.34.mlp.experts.102.gate_proj", "model.layers.34.mlp.experts.103.gate_proj", "model.layers.34.mlp.experts.104.gate_proj", "model.layers.34.mlp.experts.105.gate_proj", "model.layers.34.mlp.experts.106.gate_proj", "model.layers.34.mlp.experts.107.gate_proj", "model.layers.34.mlp.experts.108.gate_proj", "model.layers.34.mlp.experts.109.gate_proj", "model.layers.34.mlp.experts.110.gate_proj", "model.layers.34.mlp.experts.111.gate_proj", "model.layers.34.mlp.experts.112.gate_proj", "model.layers.34.mlp.experts.113.gate_proj", "model.layers.34.mlp.experts.114.gate_proj", "model.layers.34.mlp.experts.115.gate_proj", "model.layers.34.mlp.experts.116.gate_proj", "model.layers.34.mlp.experts.117.gate_proj", "model.layers.34.mlp.experts.118.gate_proj", "model.layers.34.mlp.experts.119.gate_proj", "model.layers.34.mlp.experts.120.gate_proj", "model.layers.34.mlp.experts.121.gate_proj", "model.layers.34.mlp.experts.122.gate_proj", "model.layers.34.mlp.experts.123.gate_proj", "model.layers.34.mlp.experts.124.gate_proj", "model.layers.34.mlp.experts.125.gate_proj", "model.layers.34.mlp.experts.126.gate_proj", "model.layers.34.mlp.experts.127.gate_proj", "model.layers.34.mlp.experts.128.gate_proj", "model.layers.34.mlp.experts.129.gate_proj", "model.layers.34.mlp.experts.130.gate_proj", "model.layers.34.mlp.experts.131.gate_proj", "model.layers.34.mlp.experts.132.gate_proj", "model.layers.34.mlp.experts.133.gate_proj", "model.layers.34.mlp.experts.134.gate_proj", "model.layers.34.mlp.experts.135.gate_proj", "model.layers.34.mlp.experts.136.gate_proj", "model.layers.34.mlp.experts.137.gate_proj", "model.layers.34.mlp.experts.138.gate_proj", "model.layers.34.mlp.experts.139.gate_proj", "model.layers.34.mlp.experts.140.gate_proj", "model.layers.34.mlp.experts.141.gate_proj", "model.layers.34.mlp.experts.142.gate_proj", "model.layers.34.mlp.experts.143.gate_proj", "model.layers.34.mlp.experts.144.gate_proj", "model.layers.34.mlp.experts.145.gate_proj", "model.layers.34.mlp.experts.146.gate_proj", "model.layers.34.mlp.experts.147.gate_proj", "model.layers.34.mlp.experts.148.gate_proj", "model.layers.34.mlp.experts.149.gate_proj", "model.layers.34.mlp.experts.150.gate_proj", "model.layers.34.mlp.experts.151.gate_proj", "model.layers.34.mlp.experts.152.gate_proj", "model.layers.34.mlp.experts.153.gate_proj", "model.layers.34.mlp.experts.154.gate_proj", "model.layers.34.mlp.experts.155.gate_proj", "model.layers.34.mlp.experts.156.gate_proj", "model.layers.34.mlp.experts.157.gate_proj", "model.layers.34.mlp.experts.158.gate_proj", "model.layers.34.mlp.experts.159.gate_proj", "model.layers.34.mlp.experts.0.up_proj", "model.layers.34.mlp.experts.1.up_proj", "model.layers.34.mlp.experts.2.up_proj", "model.layers.34.mlp.experts.3.up_proj", "model.layers.34.mlp.experts.4.up_proj", "model.layers.34.mlp.experts.5.up_proj", "model.layers.34.mlp.experts.6.up_proj", "model.layers.34.mlp.experts.7.up_proj", "model.layers.34.mlp.experts.8.up_proj", "model.layers.34.mlp.experts.9.up_proj", "model.layers.34.mlp.experts.10.up_proj", "model.layers.34.mlp.experts.11.up_proj", "model.layers.34.mlp.experts.12.up_proj", "model.layers.34.mlp.experts.13.up_proj", "model.layers.34.mlp.experts.14.up_proj", "model.layers.34.mlp.experts.15.up_proj", "model.layers.34.mlp.experts.16.up_proj", "model.layers.34.mlp.experts.17.up_proj", "model.layers.34.mlp.experts.18.up_proj", "model.layers.34.mlp.experts.19.up_proj", "model.layers.34.mlp.experts.20.up_proj", "model.layers.34.mlp.experts.21.up_proj", "model.layers.34.mlp.experts.22.up_proj", "model.layers.34.mlp.experts.23.up_proj", "model.layers.34.mlp.experts.24.up_proj", "model.layers.34.mlp.experts.25.up_proj", "model.layers.34.mlp.experts.26.up_proj", "model.layers.34.mlp.experts.27.up_proj", "model.layers.34.mlp.experts.28.up_proj", "model.layers.34.mlp.experts.29.up_proj", "model.layers.34.mlp.experts.30.up_proj", "model.layers.34.mlp.experts.31.up_proj", "model.layers.34.mlp.experts.32.up_proj", "model.layers.34.mlp.experts.33.up_proj", "model.layers.34.mlp.experts.34.up_proj", "model.layers.34.mlp.experts.35.up_proj", "model.layers.34.mlp.experts.36.up_proj", "model.layers.34.mlp.experts.37.up_proj", "model.layers.34.mlp.experts.38.up_proj", "model.layers.34.mlp.experts.39.up_proj", "model.layers.34.mlp.experts.40.up_proj", "model.layers.34.mlp.experts.41.up_proj", "model.layers.34.mlp.experts.42.up_proj", "model.layers.34.mlp.experts.43.up_proj", "model.layers.34.mlp.experts.44.up_proj", "model.layers.34.mlp.experts.45.up_proj", "model.layers.34.mlp.experts.46.up_proj", "model.layers.34.mlp.experts.47.up_proj", "model.layers.34.mlp.experts.48.up_proj", "model.layers.34.mlp.experts.49.up_proj", "model.layers.34.mlp.experts.50.up_proj", "model.layers.34.mlp.experts.51.up_proj", "model.layers.34.mlp.experts.52.up_proj", "model.layers.34.mlp.experts.53.up_proj", "model.layers.34.mlp.experts.54.up_proj", "model.layers.34.mlp.experts.55.up_proj", "model.layers.34.mlp.experts.56.up_proj", "model.layers.34.mlp.experts.57.up_proj", "model.layers.34.mlp.experts.58.up_proj", "model.layers.34.mlp.experts.59.up_proj", "model.layers.34.mlp.experts.60.up_proj", "model.layers.34.mlp.experts.61.up_proj", "model.layers.34.mlp.experts.62.up_proj", "model.layers.34.mlp.experts.63.up_proj", "model.layers.34.mlp.experts.64.up_proj", "model.layers.34.mlp.experts.65.up_proj", "model.layers.34.mlp.experts.66.up_proj", "model.layers.34.mlp.experts.67.up_proj", "model.layers.34.mlp.experts.68.up_proj", "model.layers.34.mlp.experts.69.up_proj", "model.layers.34.mlp.experts.70.up_proj", "model.layers.34.mlp.experts.71.up_proj", "model.layers.34.mlp.experts.72.up_proj", "model.layers.34.mlp.experts.73.up_proj", "model.layers.34.mlp.experts.74.up_proj", "model.layers.34.mlp.experts.75.up_proj", "model.layers.34.mlp.experts.76.up_proj", "model.layers.34.mlp.experts.77.up_proj", "model.layers.34.mlp.experts.78.up_proj", "model.layers.34.mlp.experts.79.up_proj", "model.layers.34.mlp.experts.80.up_proj", "model.layers.34.mlp.experts.81.up_proj", "model.layers.34.mlp.experts.82.up_proj", "model.layers.34.mlp.experts.83.up_proj", "model.layers.34.mlp.experts.84.up_proj", "model.layers.34.mlp.experts.85.up_proj", "model.layers.34.mlp.experts.86.up_proj", "model.layers.34.mlp.experts.87.up_proj", "model.layers.34.mlp.experts.88.up_proj", "model.layers.34.mlp.experts.89.up_proj", "model.layers.34.mlp.experts.90.up_proj", "model.layers.34.mlp.experts.91.up_proj", "model.layers.34.mlp.experts.92.up_proj", "model.layers.34.mlp.experts.93.up_proj", "model.layers.34.mlp.experts.94.up_proj", "model.layers.34.mlp.experts.95.up_proj", "model.layers.34.mlp.experts.96.up_proj", "model.layers.34.mlp.experts.97.up_proj", "model.layers.34.mlp.experts.98.up_proj", "model.layers.34.mlp.experts.99.up_proj", "model.layers.34.mlp.experts.100.up_proj", "model.layers.34.mlp.experts.101.up_proj", "model.layers.34.mlp.experts.102.up_proj", "model.layers.34.mlp.experts.103.up_proj", "model.layers.34.mlp.experts.104.up_proj", "model.layers.34.mlp.experts.105.up_proj", "model.layers.34.mlp.experts.106.up_proj", "model.layers.34.mlp.experts.107.up_proj", "model.layers.34.mlp.experts.108.up_proj", "model.layers.34.mlp.experts.109.up_proj", "model.layers.34.mlp.experts.110.up_proj", "model.layers.34.mlp.experts.111.up_proj", "model.layers.34.mlp.experts.112.up_proj", "model.layers.34.mlp.experts.113.up_proj", "model.layers.34.mlp.experts.114.up_proj", "model.layers.34.mlp.experts.115.up_proj", "model.layers.34.mlp.experts.116.up_proj", "model.layers.34.mlp.experts.117.up_proj", "model.layers.34.mlp.experts.118.up_proj", "model.layers.34.mlp.experts.119.up_proj", "model.layers.34.mlp.experts.120.up_proj", "model.layers.34.mlp.experts.121.up_proj", "model.layers.34.mlp.experts.122.up_proj", "model.layers.34.mlp.experts.123.up_proj", "model.layers.34.mlp.experts.124.up_proj", "model.layers.34.mlp.experts.125.up_proj", "model.layers.34.mlp.experts.126.up_proj", "model.layers.34.mlp.experts.127.up_proj", "model.layers.34.mlp.experts.128.up_proj", "model.layers.34.mlp.experts.129.up_proj", "model.layers.34.mlp.experts.130.up_proj", "model.layers.34.mlp.experts.131.up_proj", "model.layers.34.mlp.experts.132.up_proj", "model.layers.34.mlp.experts.133.up_proj", "model.layers.34.mlp.experts.134.up_proj", "model.layers.34.mlp.experts.135.up_proj", "model.layers.34.mlp.experts.136.up_proj", "model.layers.34.mlp.experts.137.up_proj", "model.layers.34.mlp.experts.138.up_proj", "model.layers.34.mlp.experts.139.up_proj", "model.layers.34.mlp.experts.140.up_proj", "model.layers.34.mlp.experts.141.up_proj", "model.layers.34.mlp.experts.142.up_proj", "model.layers.34.mlp.experts.143.up_proj", "model.layers.34.mlp.experts.144.up_proj", "model.layers.34.mlp.experts.145.up_proj", "model.layers.34.mlp.experts.146.up_proj", "model.layers.34.mlp.experts.147.up_proj", "model.layers.34.mlp.experts.148.up_proj", "model.layers.34.mlp.experts.149.up_proj", "model.layers.34.mlp.experts.150.up_proj", "model.layers.34.mlp.experts.151.up_proj", "model.layers.34.mlp.experts.152.up_proj", "model.layers.34.mlp.experts.153.up_proj", "model.layers.34.mlp.experts.154.up_proj", "model.layers.34.mlp.experts.155.up_proj", "model.layers.34.mlp.experts.156.up_proj", "model.layers.34.mlp.experts.157.up_proj", "model.layers.34.mlp.experts.158.up_proj", "model.layers.34.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.250708687119186e-06, "dbits": 5033164800 } ] }, { "idx": 206, "layers": [ "model.layers.34.mlp.experts.0.down_proj", "model.layers.34.mlp.experts.1.down_proj", "model.layers.34.mlp.experts.2.down_proj", "model.layers.34.mlp.experts.3.down_proj", "model.layers.34.mlp.experts.4.down_proj", "model.layers.34.mlp.experts.5.down_proj", "model.layers.34.mlp.experts.6.down_proj", "model.layers.34.mlp.experts.7.down_proj", "model.layers.34.mlp.experts.8.down_proj", "model.layers.34.mlp.experts.9.down_proj", "model.layers.34.mlp.experts.10.down_proj", "model.layers.34.mlp.experts.11.down_proj", "model.layers.34.mlp.experts.12.down_proj", "model.layers.34.mlp.experts.13.down_proj", "model.layers.34.mlp.experts.14.down_proj", "model.layers.34.mlp.experts.15.down_proj", "model.layers.34.mlp.experts.16.down_proj", "model.layers.34.mlp.experts.17.down_proj", "model.layers.34.mlp.experts.18.down_proj", "model.layers.34.mlp.experts.19.down_proj", "model.layers.34.mlp.experts.20.down_proj", "model.layers.34.mlp.experts.21.down_proj", "model.layers.34.mlp.experts.22.down_proj", "model.layers.34.mlp.experts.23.down_proj", "model.layers.34.mlp.experts.24.down_proj", "model.layers.34.mlp.experts.25.down_proj", "model.layers.34.mlp.experts.26.down_proj", "model.layers.34.mlp.experts.27.down_proj", "model.layers.34.mlp.experts.28.down_proj", "model.layers.34.mlp.experts.29.down_proj", "model.layers.34.mlp.experts.30.down_proj", "model.layers.34.mlp.experts.31.down_proj", "model.layers.34.mlp.experts.32.down_proj", "model.layers.34.mlp.experts.33.down_proj", "model.layers.34.mlp.experts.34.down_proj", "model.layers.34.mlp.experts.35.down_proj", "model.layers.34.mlp.experts.36.down_proj", "model.layers.34.mlp.experts.37.down_proj", "model.layers.34.mlp.experts.38.down_proj", "model.layers.34.mlp.experts.39.down_proj", "model.layers.34.mlp.experts.40.down_proj", "model.layers.34.mlp.experts.41.down_proj", "model.layers.34.mlp.experts.42.down_proj", "model.layers.34.mlp.experts.43.down_proj", "model.layers.34.mlp.experts.44.down_proj", "model.layers.34.mlp.experts.45.down_proj", "model.layers.34.mlp.experts.46.down_proj", "model.layers.34.mlp.experts.47.down_proj", "model.layers.34.mlp.experts.48.down_proj", "model.layers.34.mlp.experts.49.down_proj", "model.layers.34.mlp.experts.50.down_proj", "model.layers.34.mlp.experts.51.down_proj", "model.layers.34.mlp.experts.52.down_proj", "model.layers.34.mlp.experts.53.down_proj", "model.layers.34.mlp.experts.54.down_proj", "model.layers.34.mlp.experts.55.down_proj", "model.layers.34.mlp.experts.56.down_proj", "model.layers.34.mlp.experts.57.down_proj", "model.layers.34.mlp.experts.58.down_proj", "model.layers.34.mlp.experts.59.down_proj", "model.layers.34.mlp.experts.60.down_proj", "model.layers.34.mlp.experts.61.down_proj", "model.layers.34.mlp.experts.62.down_proj", "model.layers.34.mlp.experts.63.down_proj", "model.layers.34.mlp.experts.64.down_proj", "model.layers.34.mlp.experts.65.down_proj", "model.layers.34.mlp.experts.66.down_proj", "model.layers.34.mlp.experts.67.down_proj", "model.layers.34.mlp.experts.68.down_proj", "model.layers.34.mlp.experts.69.down_proj", "model.layers.34.mlp.experts.70.down_proj", "model.layers.34.mlp.experts.71.down_proj", "model.layers.34.mlp.experts.72.down_proj", "model.layers.34.mlp.experts.73.down_proj", "model.layers.34.mlp.experts.74.down_proj", "model.layers.34.mlp.experts.75.down_proj", "model.layers.34.mlp.experts.76.down_proj", "model.layers.34.mlp.experts.77.down_proj", "model.layers.34.mlp.experts.78.down_proj", "model.layers.34.mlp.experts.79.down_proj", "model.layers.34.mlp.experts.80.down_proj", "model.layers.34.mlp.experts.81.down_proj", "model.layers.34.mlp.experts.82.down_proj", "model.layers.34.mlp.experts.83.down_proj", "model.layers.34.mlp.experts.84.down_proj", "model.layers.34.mlp.experts.85.down_proj", "model.layers.34.mlp.experts.86.down_proj", "model.layers.34.mlp.experts.87.down_proj", "model.layers.34.mlp.experts.88.down_proj", "model.layers.34.mlp.experts.89.down_proj", "model.layers.34.mlp.experts.90.down_proj", "model.layers.34.mlp.experts.91.down_proj", "model.layers.34.mlp.experts.92.down_proj", "model.layers.34.mlp.experts.93.down_proj", "model.layers.34.mlp.experts.94.down_proj", "model.layers.34.mlp.experts.95.down_proj", "model.layers.34.mlp.experts.96.down_proj", "model.layers.34.mlp.experts.97.down_proj", "model.layers.34.mlp.experts.98.down_proj", "model.layers.34.mlp.experts.99.down_proj", "model.layers.34.mlp.experts.100.down_proj", "model.layers.34.mlp.experts.101.down_proj", "model.layers.34.mlp.experts.102.down_proj", "model.layers.34.mlp.experts.103.down_proj", "model.layers.34.mlp.experts.104.down_proj", "model.layers.34.mlp.experts.105.down_proj", "model.layers.34.mlp.experts.106.down_proj", "model.layers.34.mlp.experts.107.down_proj", "model.layers.34.mlp.experts.108.down_proj", "model.layers.34.mlp.experts.109.down_proj", "model.layers.34.mlp.experts.110.down_proj", "model.layers.34.mlp.experts.111.down_proj", "model.layers.34.mlp.experts.112.down_proj", "model.layers.34.mlp.experts.113.down_proj", "model.layers.34.mlp.experts.114.down_proj", "model.layers.34.mlp.experts.115.down_proj", "model.layers.34.mlp.experts.116.down_proj", "model.layers.34.mlp.experts.117.down_proj", "model.layers.34.mlp.experts.118.down_proj", "model.layers.34.mlp.experts.119.down_proj", "model.layers.34.mlp.experts.120.down_proj", "model.layers.34.mlp.experts.121.down_proj", "model.layers.34.mlp.experts.122.down_proj", "model.layers.34.mlp.experts.123.down_proj", "model.layers.34.mlp.experts.124.down_proj", "model.layers.34.mlp.experts.125.down_proj", "model.layers.34.mlp.experts.126.down_proj", "model.layers.34.mlp.experts.127.down_proj", "model.layers.34.mlp.experts.128.down_proj", "model.layers.34.mlp.experts.129.down_proj", "model.layers.34.mlp.experts.130.down_proj", "model.layers.34.mlp.experts.131.down_proj", "model.layers.34.mlp.experts.132.down_proj", "model.layers.34.mlp.experts.133.down_proj", "model.layers.34.mlp.experts.134.down_proj", "model.layers.34.mlp.experts.135.down_proj", "model.layers.34.mlp.experts.136.down_proj", "model.layers.34.mlp.experts.137.down_proj", "model.layers.34.mlp.experts.138.down_proj", "model.layers.34.mlp.experts.139.down_proj", "model.layers.34.mlp.experts.140.down_proj", "model.layers.34.mlp.experts.141.down_proj", "model.layers.34.mlp.experts.142.down_proj", "model.layers.34.mlp.experts.143.down_proj", "model.layers.34.mlp.experts.144.down_proj", "model.layers.34.mlp.experts.145.down_proj", "model.layers.34.mlp.experts.146.down_proj", "model.layers.34.mlp.experts.147.down_proj", "model.layers.34.mlp.experts.148.down_proj", "model.layers.34.mlp.experts.149.down_proj", "model.layers.34.mlp.experts.150.down_proj", "model.layers.34.mlp.experts.151.down_proj", "model.layers.34.mlp.experts.152.down_proj", "model.layers.34.mlp.experts.153.down_proj", "model.layers.34.mlp.experts.154.down_proj", "model.layers.34.mlp.experts.155.down_proj", "model.layers.34.mlp.experts.156.down_proj", "model.layers.34.mlp.experts.157.down_proj", "model.layers.34.mlp.experts.158.down_proj", "model.layers.34.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.931938459165351e-06, "dbits": 2516582400 } ] }, { "idx": 207, "layers": [ "model.layers.35.self_attn.q_proj" ], "candidates": [ { "dkld": -8.794595487417959e-08, "dbits": 125829120 } ] }, { "idx": 208, "layers": [ "model.layers.35.self_attn.k_proj", "model.layers.35.self_attn.v_proj" ], "candidates": [ { "dkld": -3.182361251674636e-06, "dbits": 20971520 } ] }, { "idx": 209, "layers": [ "model.layers.35.self_attn.o_proj" ], "candidates": [ { "dkld": -1.031639985740187e-05, "dbits": 125829120 } ] }, { "idx": 210, "layers": [ "model.layers.35.mlp.shared_experts.gate_proj", "model.layers.35.mlp.shared_experts.up_proj", "model.layers.35.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.3165648346767046e-06, "dbits": 47185920 } ] }, { "idx": 211, "layers": [ "model.layers.35.mlp.experts.0.gate_proj", "model.layers.35.mlp.experts.1.gate_proj", "model.layers.35.mlp.experts.2.gate_proj", "model.layers.35.mlp.experts.3.gate_proj", "model.layers.35.mlp.experts.4.gate_proj", "model.layers.35.mlp.experts.5.gate_proj", "model.layers.35.mlp.experts.6.gate_proj", "model.layers.35.mlp.experts.7.gate_proj", "model.layers.35.mlp.experts.8.gate_proj", "model.layers.35.mlp.experts.9.gate_proj", "model.layers.35.mlp.experts.10.gate_proj", "model.layers.35.mlp.experts.11.gate_proj", "model.layers.35.mlp.experts.12.gate_proj", "model.layers.35.mlp.experts.13.gate_proj", "model.layers.35.mlp.experts.14.gate_proj", "model.layers.35.mlp.experts.15.gate_proj", "model.layers.35.mlp.experts.16.gate_proj", "model.layers.35.mlp.experts.17.gate_proj", "model.layers.35.mlp.experts.18.gate_proj", "model.layers.35.mlp.experts.19.gate_proj", "model.layers.35.mlp.experts.20.gate_proj", "model.layers.35.mlp.experts.21.gate_proj", "model.layers.35.mlp.experts.22.gate_proj", "model.layers.35.mlp.experts.23.gate_proj", "model.layers.35.mlp.experts.24.gate_proj", "model.layers.35.mlp.experts.25.gate_proj", "model.layers.35.mlp.experts.26.gate_proj", "model.layers.35.mlp.experts.27.gate_proj", "model.layers.35.mlp.experts.28.gate_proj", "model.layers.35.mlp.experts.29.gate_proj", "model.layers.35.mlp.experts.30.gate_proj", "model.layers.35.mlp.experts.31.gate_proj", "model.layers.35.mlp.experts.32.gate_proj", "model.layers.35.mlp.experts.33.gate_proj", "model.layers.35.mlp.experts.34.gate_proj", "model.layers.35.mlp.experts.35.gate_proj", "model.layers.35.mlp.experts.36.gate_proj", "model.layers.35.mlp.experts.37.gate_proj", "model.layers.35.mlp.experts.38.gate_proj", "model.layers.35.mlp.experts.39.gate_proj", "model.layers.35.mlp.experts.40.gate_proj", "model.layers.35.mlp.experts.41.gate_proj", "model.layers.35.mlp.experts.42.gate_proj", "model.layers.35.mlp.experts.43.gate_proj", "model.layers.35.mlp.experts.44.gate_proj", "model.layers.35.mlp.experts.45.gate_proj", "model.layers.35.mlp.experts.46.gate_proj", "model.layers.35.mlp.experts.47.gate_proj", "model.layers.35.mlp.experts.48.gate_proj", "model.layers.35.mlp.experts.49.gate_proj", "model.layers.35.mlp.experts.50.gate_proj", "model.layers.35.mlp.experts.51.gate_proj", "model.layers.35.mlp.experts.52.gate_proj", "model.layers.35.mlp.experts.53.gate_proj", "model.layers.35.mlp.experts.54.gate_proj", "model.layers.35.mlp.experts.55.gate_proj", "model.layers.35.mlp.experts.56.gate_proj", "model.layers.35.mlp.experts.57.gate_proj", "model.layers.35.mlp.experts.58.gate_proj", "model.layers.35.mlp.experts.59.gate_proj", "model.layers.35.mlp.experts.60.gate_proj", "model.layers.35.mlp.experts.61.gate_proj", "model.layers.35.mlp.experts.62.gate_proj", "model.layers.35.mlp.experts.63.gate_proj", "model.layers.35.mlp.experts.64.gate_proj", "model.layers.35.mlp.experts.65.gate_proj", "model.layers.35.mlp.experts.66.gate_proj", "model.layers.35.mlp.experts.67.gate_proj", "model.layers.35.mlp.experts.68.gate_proj", "model.layers.35.mlp.experts.69.gate_proj", "model.layers.35.mlp.experts.70.gate_proj", "model.layers.35.mlp.experts.71.gate_proj", "model.layers.35.mlp.experts.72.gate_proj", "model.layers.35.mlp.experts.73.gate_proj", "model.layers.35.mlp.experts.74.gate_proj", "model.layers.35.mlp.experts.75.gate_proj", "model.layers.35.mlp.experts.76.gate_proj", "model.layers.35.mlp.experts.77.gate_proj", "model.layers.35.mlp.experts.78.gate_proj", "model.layers.35.mlp.experts.79.gate_proj", "model.layers.35.mlp.experts.80.gate_proj", "model.layers.35.mlp.experts.81.gate_proj", "model.layers.35.mlp.experts.82.gate_proj", "model.layers.35.mlp.experts.83.gate_proj", "model.layers.35.mlp.experts.84.gate_proj", "model.layers.35.mlp.experts.85.gate_proj", "model.layers.35.mlp.experts.86.gate_proj", "model.layers.35.mlp.experts.87.gate_proj", "model.layers.35.mlp.experts.88.gate_proj", "model.layers.35.mlp.experts.89.gate_proj", "model.layers.35.mlp.experts.90.gate_proj", "model.layers.35.mlp.experts.91.gate_proj", "model.layers.35.mlp.experts.92.gate_proj", "model.layers.35.mlp.experts.93.gate_proj", "model.layers.35.mlp.experts.94.gate_proj", "model.layers.35.mlp.experts.95.gate_proj", "model.layers.35.mlp.experts.96.gate_proj", "model.layers.35.mlp.experts.97.gate_proj", "model.layers.35.mlp.experts.98.gate_proj", "model.layers.35.mlp.experts.99.gate_proj", "model.layers.35.mlp.experts.100.gate_proj", "model.layers.35.mlp.experts.101.gate_proj", "model.layers.35.mlp.experts.102.gate_proj", "model.layers.35.mlp.experts.103.gate_proj", "model.layers.35.mlp.experts.104.gate_proj", "model.layers.35.mlp.experts.105.gate_proj", "model.layers.35.mlp.experts.106.gate_proj", "model.layers.35.mlp.experts.107.gate_proj", "model.layers.35.mlp.experts.108.gate_proj", "model.layers.35.mlp.experts.109.gate_proj", "model.layers.35.mlp.experts.110.gate_proj", "model.layers.35.mlp.experts.111.gate_proj", "model.layers.35.mlp.experts.112.gate_proj", "model.layers.35.mlp.experts.113.gate_proj", "model.layers.35.mlp.experts.114.gate_proj", "model.layers.35.mlp.experts.115.gate_proj", "model.layers.35.mlp.experts.116.gate_proj", "model.layers.35.mlp.experts.117.gate_proj", "model.layers.35.mlp.experts.118.gate_proj", "model.layers.35.mlp.experts.119.gate_proj", "model.layers.35.mlp.experts.120.gate_proj", "model.layers.35.mlp.experts.121.gate_proj", "model.layers.35.mlp.experts.122.gate_proj", "model.layers.35.mlp.experts.123.gate_proj", "model.layers.35.mlp.experts.124.gate_proj", "model.layers.35.mlp.experts.125.gate_proj", "model.layers.35.mlp.experts.126.gate_proj", "model.layers.35.mlp.experts.127.gate_proj", "model.layers.35.mlp.experts.128.gate_proj", "model.layers.35.mlp.experts.129.gate_proj", "model.layers.35.mlp.experts.130.gate_proj", "model.layers.35.mlp.experts.131.gate_proj", "model.layers.35.mlp.experts.132.gate_proj", "model.layers.35.mlp.experts.133.gate_proj", "model.layers.35.mlp.experts.134.gate_proj", "model.layers.35.mlp.experts.135.gate_proj", "model.layers.35.mlp.experts.136.gate_proj", "model.layers.35.mlp.experts.137.gate_proj", "model.layers.35.mlp.experts.138.gate_proj", "model.layers.35.mlp.experts.139.gate_proj", "model.layers.35.mlp.experts.140.gate_proj", "model.layers.35.mlp.experts.141.gate_proj", "model.layers.35.mlp.experts.142.gate_proj", "model.layers.35.mlp.experts.143.gate_proj", "model.layers.35.mlp.experts.144.gate_proj", "model.layers.35.mlp.experts.145.gate_proj", "model.layers.35.mlp.experts.146.gate_proj", "model.layers.35.mlp.experts.147.gate_proj", "model.layers.35.mlp.experts.148.gate_proj", "model.layers.35.mlp.experts.149.gate_proj", "model.layers.35.mlp.experts.150.gate_proj", "model.layers.35.mlp.experts.151.gate_proj", "model.layers.35.mlp.experts.152.gate_proj", "model.layers.35.mlp.experts.153.gate_proj", "model.layers.35.mlp.experts.154.gate_proj", "model.layers.35.mlp.experts.155.gate_proj", "model.layers.35.mlp.experts.156.gate_proj", "model.layers.35.mlp.experts.157.gate_proj", "model.layers.35.mlp.experts.158.gate_proj", "model.layers.35.mlp.experts.159.gate_proj", "model.layers.35.mlp.experts.0.up_proj", "model.layers.35.mlp.experts.1.up_proj", "model.layers.35.mlp.experts.2.up_proj", "model.layers.35.mlp.experts.3.up_proj", "model.layers.35.mlp.experts.4.up_proj", "model.layers.35.mlp.experts.5.up_proj", "model.layers.35.mlp.experts.6.up_proj", "model.layers.35.mlp.experts.7.up_proj", "model.layers.35.mlp.experts.8.up_proj", "model.layers.35.mlp.experts.9.up_proj", "model.layers.35.mlp.experts.10.up_proj", "model.layers.35.mlp.experts.11.up_proj", "model.layers.35.mlp.experts.12.up_proj", "model.layers.35.mlp.experts.13.up_proj", "model.layers.35.mlp.experts.14.up_proj", "model.layers.35.mlp.experts.15.up_proj", "model.layers.35.mlp.experts.16.up_proj", "model.layers.35.mlp.experts.17.up_proj", "model.layers.35.mlp.experts.18.up_proj", "model.layers.35.mlp.experts.19.up_proj", "model.layers.35.mlp.experts.20.up_proj", "model.layers.35.mlp.experts.21.up_proj", "model.layers.35.mlp.experts.22.up_proj", "model.layers.35.mlp.experts.23.up_proj", "model.layers.35.mlp.experts.24.up_proj", "model.layers.35.mlp.experts.25.up_proj", "model.layers.35.mlp.experts.26.up_proj", "model.layers.35.mlp.experts.27.up_proj", "model.layers.35.mlp.experts.28.up_proj", "model.layers.35.mlp.experts.29.up_proj", "model.layers.35.mlp.experts.30.up_proj", "model.layers.35.mlp.experts.31.up_proj", "model.layers.35.mlp.experts.32.up_proj", "model.layers.35.mlp.experts.33.up_proj", "model.layers.35.mlp.experts.34.up_proj", "model.layers.35.mlp.experts.35.up_proj", "model.layers.35.mlp.experts.36.up_proj", "model.layers.35.mlp.experts.37.up_proj", "model.layers.35.mlp.experts.38.up_proj", "model.layers.35.mlp.experts.39.up_proj", "model.layers.35.mlp.experts.40.up_proj", "model.layers.35.mlp.experts.41.up_proj", "model.layers.35.mlp.experts.42.up_proj", "model.layers.35.mlp.experts.43.up_proj", "model.layers.35.mlp.experts.44.up_proj", "model.layers.35.mlp.experts.45.up_proj", "model.layers.35.mlp.experts.46.up_proj", "model.layers.35.mlp.experts.47.up_proj", "model.layers.35.mlp.experts.48.up_proj", "model.layers.35.mlp.experts.49.up_proj", "model.layers.35.mlp.experts.50.up_proj", "model.layers.35.mlp.experts.51.up_proj", "model.layers.35.mlp.experts.52.up_proj", "model.layers.35.mlp.experts.53.up_proj", "model.layers.35.mlp.experts.54.up_proj", "model.layers.35.mlp.experts.55.up_proj", "model.layers.35.mlp.experts.56.up_proj", "model.layers.35.mlp.experts.57.up_proj", "model.layers.35.mlp.experts.58.up_proj", "model.layers.35.mlp.experts.59.up_proj", "model.layers.35.mlp.experts.60.up_proj", "model.layers.35.mlp.experts.61.up_proj", "model.layers.35.mlp.experts.62.up_proj", "model.layers.35.mlp.experts.63.up_proj", "model.layers.35.mlp.experts.64.up_proj", "model.layers.35.mlp.experts.65.up_proj", "model.layers.35.mlp.experts.66.up_proj", "model.layers.35.mlp.experts.67.up_proj", "model.layers.35.mlp.experts.68.up_proj", "model.layers.35.mlp.experts.69.up_proj", "model.layers.35.mlp.experts.70.up_proj", "model.layers.35.mlp.experts.71.up_proj", "model.layers.35.mlp.experts.72.up_proj", "model.layers.35.mlp.experts.73.up_proj", "model.layers.35.mlp.experts.74.up_proj", "model.layers.35.mlp.experts.75.up_proj", "model.layers.35.mlp.experts.76.up_proj", "model.layers.35.mlp.experts.77.up_proj", "model.layers.35.mlp.experts.78.up_proj", "model.layers.35.mlp.experts.79.up_proj", "model.layers.35.mlp.experts.80.up_proj", "model.layers.35.mlp.experts.81.up_proj", "model.layers.35.mlp.experts.82.up_proj", "model.layers.35.mlp.experts.83.up_proj", "model.layers.35.mlp.experts.84.up_proj", "model.layers.35.mlp.experts.85.up_proj", "model.layers.35.mlp.experts.86.up_proj", "model.layers.35.mlp.experts.87.up_proj", "model.layers.35.mlp.experts.88.up_proj", "model.layers.35.mlp.experts.89.up_proj", "model.layers.35.mlp.experts.90.up_proj", "model.layers.35.mlp.experts.91.up_proj", "model.layers.35.mlp.experts.92.up_proj", "model.layers.35.mlp.experts.93.up_proj", "model.layers.35.mlp.experts.94.up_proj", "model.layers.35.mlp.experts.95.up_proj", "model.layers.35.mlp.experts.96.up_proj", "model.layers.35.mlp.experts.97.up_proj", "model.layers.35.mlp.experts.98.up_proj", "model.layers.35.mlp.experts.99.up_proj", "model.layers.35.mlp.experts.100.up_proj", "model.layers.35.mlp.experts.101.up_proj", "model.layers.35.mlp.experts.102.up_proj", "model.layers.35.mlp.experts.103.up_proj", "model.layers.35.mlp.experts.104.up_proj", "model.layers.35.mlp.experts.105.up_proj", "model.layers.35.mlp.experts.106.up_proj", "model.layers.35.mlp.experts.107.up_proj", "model.layers.35.mlp.experts.108.up_proj", "model.layers.35.mlp.experts.109.up_proj", "model.layers.35.mlp.experts.110.up_proj", "model.layers.35.mlp.experts.111.up_proj", "model.layers.35.mlp.experts.112.up_proj", "model.layers.35.mlp.experts.113.up_proj", "model.layers.35.mlp.experts.114.up_proj", "model.layers.35.mlp.experts.115.up_proj", "model.layers.35.mlp.experts.116.up_proj", "model.layers.35.mlp.experts.117.up_proj", "model.layers.35.mlp.experts.118.up_proj", "model.layers.35.mlp.experts.119.up_proj", "model.layers.35.mlp.experts.120.up_proj", "model.layers.35.mlp.experts.121.up_proj", "model.layers.35.mlp.experts.122.up_proj", "model.layers.35.mlp.experts.123.up_proj", "model.layers.35.mlp.experts.124.up_proj", "model.layers.35.mlp.experts.125.up_proj", "model.layers.35.mlp.experts.126.up_proj", "model.layers.35.mlp.experts.127.up_proj", "model.layers.35.mlp.experts.128.up_proj", "model.layers.35.mlp.experts.129.up_proj", "model.layers.35.mlp.experts.130.up_proj", "model.layers.35.mlp.experts.131.up_proj", "model.layers.35.mlp.experts.132.up_proj", "model.layers.35.mlp.experts.133.up_proj", "model.layers.35.mlp.experts.134.up_proj", "model.layers.35.mlp.experts.135.up_proj", "model.layers.35.mlp.experts.136.up_proj", "model.layers.35.mlp.experts.137.up_proj", "model.layers.35.mlp.experts.138.up_proj", "model.layers.35.mlp.experts.139.up_proj", "model.layers.35.mlp.experts.140.up_proj", "model.layers.35.mlp.experts.141.up_proj", "model.layers.35.mlp.experts.142.up_proj", "model.layers.35.mlp.experts.143.up_proj", "model.layers.35.mlp.experts.144.up_proj", "model.layers.35.mlp.experts.145.up_proj", "model.layers.35.mlp.experts.146.up_proj", "model.layers.35.mlp.experts.147.up_proj", "model.layers.35.mlp.experts.148.up_proj", "model.layers.35.mlp.experts.149.up_proj", "model.layers.35.mlp.experts.150.up_proj", "model.layers.35.mlp.experts.151.up_proj", "model.layers.35.mlp.experts.152.up_proj", "model.layers.35.mlp.experts.153.up_proj", "model.layers.35.mlp.experts.154.up_proj", "model.layers.35.mlp.experts.155.up_proj", "model.layers.35.mlp.experts.156.up_proj", "model.layers.35.mlp.experts.157.up_proj", "model.layers.35.mlp.experts.158.up_proj", "model.layers.35.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.033949855715123e-06, "dbits": 5033164800 } ] }, { "idx": 212, "layers": [ "model.layers.35.mlp.experts.0.down_proj", "model.layers.35.mlp.experts.1.down_proj", "model.layers.35.mlp.experts.2.down_proj", "model.layers.35.mlp.experts.3.down_proj", "model.layers.35.mlp.experts.4.down_proj", "model.layers.35.mlp.experts.5.down_proj", "model.layers.35.mlp.experts.6.down_proj", "model.layers.35.mlp.experts.7.down_proj", "model.layers.35.mlp.experts.8.down_proj", "model.layers.35.mlp.experts.9.down_proj", "model.layers.35.mlp.experts.10.down_proj", "model.layers.35.mlp.experts.11.down_proj", "model.layers.35.mlp.experts.12.down_proj", "model.layers.35.mlp.experts.13.down_proj", "model.layers.35.mlp.experts.14.down_proj", "model.layers.35.mlp.experts.15.down_proj", "model.layers.35.mlp.experts.16.down_proj", "model.layers.35.mlp.experts.17.down_proj", "model.layers.35.mlp.experts.18.down_proj", "model.layers.35.mlp.experts.19.down_proj", "model.layers.35.mlp.experts.20.down_proj", "model.layers.35.mlp.experts.21.down_proj", "model.layers.35.mlp.experts.22.down_proj", "model.layers.35.mlp.experts.23.down_proj", "model.layers.35.mlp.experts.24.down_proj", "model.layers.35.mlp.experts.25.down_proj", "model.layers.35.mlp.experts.26.down_proj", "model.layers.35.mlp.experts.27.down_proj", "model.layers.35.mlp.experts.28.down_proj", "model.layers.35.mlp.experts.29.down_proj", "model.layers.35.mlp.experts.30.down_proj", "model.layers.35.mlp.experts.31.down_proj", "model.layers.35.mlp.experts.32.down_proj", "model.layers.35.mlp.experts.33.down_proj", "model.layers.35.mlp.experts.34.down_proj", "model.layers.35.mlp.experts.35.down_proj", "model.layers.35.mlp.experts.36.down_proj", "model.layers.35.mlp.experts.37.down_proj", "model.layers.35.mlp.experts.38.down_proj", "model.layers.35.mlp.experts.39.down_proj", "model.layers.35.mlp.experts.40.down_proj", "model.layers.35.mlp.experts.41.down_proj", "model.layers.35.mlp.experts.42.down_proj", "model.layers.35.mlp.experts.43.down_proj", "model.layers.35.mlp.experts.44.down_proj", "model.layers.35.mlp.experts.45.down_proj", "model.layers.35.mlp.experts.46.down_proj", "model.layers.35.mlp.experts.47.down_proj", "model.layers.35.mlp.experts.48.down_proj", "model.layers.35.mlp.experts.49.down_proj", "model.layers.35.mlp.experts.50.down_proj", "model.layers.35.mlp.experts.51.down_proj", "model.layers.35.mlp.experts.52.down_proj", "model.layers.35.mlp.experts.53.down_proj", "model.layers.35.mlp.experts.54.down_proj", "model.layers.35.mlp.experts.55.down_proj", "model.layers.35.mlp.experts.56.down_proj", "model.layers.35.mlp.experts.57.down_proj", "model.layers.35.mlp.experts.58.down_proj", "model.layers.35.mlp.experts.59.down_proj", "model.layers.35.mlp.experts.60.down_proj", "model.layers.35.mlp.experts.61.down_proj", "model.layers.35.mlp.experts.62.down_proj", "model.layers.35.mlp.experts.63.down_proj", "model.layers.35.mlp.experts.64.down_proj", "model.layers.35.mlp.experts.65.down_proj", "model.layers.35.mlp.experts.66.down_proj", "model.layers.35.mlp.experts.67.down_proj", "model.layers.35.mlp.experts.68.down_proj", "model.layers.35.mlp.experts.69.down_proj", "model.layers.35.mlp.experts.70.down_proj", "model.layers.35.mlp.experts.71.down_proj", "model.layers.35.mlp.experts.72.down_proj", "model.layers.35.mlp.experts.73.down_proj", "model.layers.35.mlp.experts.74.down_proj", "model.layers.35.mlp.experts.75.down_proj", "model.layers.35.mlp.experts.76.down_proj", "model.layers.35.mlp.experts.77.down_proj", "model.layers.35.mlp.experts.78.down_proj", "model.layers.35.mlp.experts.79.down_proj", "model.layers.35.mlp.experts.80.down_proj", "model.layers.35.mlp.experts.81.down_proj", "model.layers.35.mlp.experts.82.down_proj", "model.layers.35.mlp.experts.83.down_proj", "model.layers.35.mlp.experts.84.down_proj", "model.layers.35.mlp.experts.85.down_proj", "model.layers.35.mlp.experts.86.down_proj", "model.layers.35.mlp.experts.87.down_proj", "model.layers.35.mlp.experts.88.down_proj", "model.layers.35.mlp.experts.89.down_proj", "model.layers.35.mlp.experts.90.down_proj", "model.layers.35.mlp.experts.91.down_proj", "model.layers.35.mlp.experts.92.down_proj", "model.layers.35.mlp.experts.93.down_proj", "model.layers.35.mlp.experts.94.down_proj", "model.layers.35.mlp.experts.95.down_proj", "model.layers.35.mlp.experts.96.down_proj", "model.layers.35.mlp.experts.97.down_proj", "model.layers.35.mlp.experts.98.down_proj", "model.layers.35.mlp.experts.99.down_proj", "model.layers.35.mlp.experts.100.down_proj", "model.layers.35.mlp.experts.101.down_proj", "model.layers.35.mlp.experts.102.down_proj", "model.layers.35.mlp.experts.103.down_proj", "model.layers.35.mlp.experts.104.down_proj", "model.layers.35.mlp.experts.105.down_proj", "model.layers.35.mlp.experts.106.down_proj", "model.layers.35.mlp.experts.107.down_proj", "model.layers.35.mlp.experts.108.down_proj", "model.layers.35.mlp.experts.109.down_proj", "model.layers.35.mlp.experts.110.down_proj", "model.layers.35.mlp.experts.111.down_proj", "model.layers.35.mlp.experts.112.down_proj", "model.layers.35.mlp.experts.113.down_proj", "model.layers.35.mlp.experts.114.down_proj", "model.layers.35.mlp.experts.115.down_proj", "model.layers.35.mlp.experts.116.down_proj", "model.layers.35.mlp.experts.117.down_proj", "model.layers.35.mlp.experts.118.down_proj", "model.layers.35.mlp.experts.119.down_proj", "model.layers.35.mlp.experts.120.down_proj", "model.layers.35.mlp.experts.121.down_proj", "model.layers.35.mlp.experts.122.down_proj", "model.layers.35.mlp.experts.123.down_proj", "model.layers.35.mlp.experts.124.down_proj", "model.layers.35.mlp.experts.125.down_proj", "model.layers.35.mlp.experts.126.down_proj", "model.layers.35.mlp.experts.127.down_proj", "model.layers.35.mlp.experts.128.down_proj", "model.layers.35.mlp.experts.129.down_proj", "model.layers.35.mlp.experts.130.down_proj", "model.layers.35.mlp.experts.131.down_proj", "model.layers.35.mlp.experts.132.down_proj", "model.layers.35.mlp.experts.133.down_proj", "model.layers.35.mlp.experts.134.down_proj", "model.layers.35.mlp.experts.135.down_proj", "model.layers.35.mlp.experts.136.down_proj", "model.layers.35.mlp.experts.137.down_proj", "model.layers.35.mlp.experts.138.down_proj", "model.layers.35.mlp.experts.139.down_proj", "model.layers.35.mlp.experts.140.down_proj", "model.layers.35.mlp.experts.141.down_proj", "model.layers.35.mlp.experts.142.down_proj", "model.layers.35.mlp.experts.143.down_proj", "model.layers.35.mlp.experts.144.down_proj", "model.layers.35.mlp.experts.145.down_proj", "model.layers.35.mlp.experts.146.down_proj", "model.layers.35.mlp.experts.147.down_proj", "model.layers.35.mlp.experts.148.down_proj", "model.layers.35.mlp.experts.149.down_proj", "model.layers.35.mlp.experts.150.down_proj", "model.layers.35.mlp.experts.151.down_proj", "model.layers.35.mlp.experts.152.down_proj", "model.layers.35.mlp.experts.153.down_proj", "model.layers.35.mlp.experts.154.down_proj", "model.layers.35.mlp.experts.155.down_proj", "model.layers.35.mlp.experts.156.down_proj", "model.layers.35.mlp.experts.157.down_proj", "model.layers.35.mlp.experts.158.down_proj", "model.layers.35.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.1322466889396093e-06, "dbits": 2516582400 } ] }, { "idx": 213, "layers": [ "model.layers.36.self_attn.q_proj" ], "candidates": [ { "dkld": -1.913600135594628e-06, "dbits": 125829120 } ] }, { "idx": 214, "layers": [ "model.layers.36.self_attn.k_proj", "model.layers.36.self_attn.v_proj" ], "candidates": [ { "dkld": -5.051406333222985e-07, "dbits": 20971520 } ] }, { "idx": 215, "layers": [ "model.layers.36.self_attn.o_proj" ], "candidates": [ { "dkld": -9.061128366738558e-06, "dbits": 125829120 } ] }, { "idx": 216, "layers": [ "model.layers.36.mlp.shared_experts.gate_proj", "model.layers.36.mlp.shared_experts.up_proj", "model.layers.36.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.0049910633824787e-05, "dbits": 47185920 } ] }, { "idx": 217, "layers": [ "model.layers.36.mlp.experts.0.gate_proj", "model.layers.36.mlp.experts.1.gate_proj", "model.layers.36.mlp.experts.2.gate_proj", "model.layers.36.mlp.experts.3.gate_proj", "model.layers.36.mlp.experts.4.gate_proj", "model.layers.36.mlp.experts.5.gate_proj", "model.layers.36.mlp.experts.6.gate_proj", "model.layers.36.mlp.experts.7.gate_proj", "model.layers.36.mlp.experts.8.gate_proj", "model.layers.36.mlp.experts.9.gate_proj", "model.layers.36.mlp.experts.10.gate_proj", "model.layers.36.mlp.experts.11.gate_proj", "model.layers.36.mlp.experts.12.gate_proj", "model.layers.36.mlp.experts.13.gate_proj", "model.layers.36.mlp.experts.14.gate_proj", "model.layers.36.mlp.experts.15.gate_proj", "model.layers.36.mlp.experts.16.gate_proj", "model.layers.36.mlp.experts.17.gate_proj", "model.layers.36.mlp.experts.18.gate_proj", "model.layers.36.mlp.experts.19.gate_proj", "model.layers.36.mlp.experts.20.gate_proj", "model.layers.36.mlp.experts.21.gate_proj", "model.layers.36.mlp.experts.22.gate_proj", "model.layers.36.mlp.experts.23.gate_proj", "model.layers.36.mlp.experts.24.gate_proj", "model.layers.36.mlp.experts.25.gate_proj", "model.layers.36.mlp.experts.26.gate_proj", "model.layers.36.mlp.experts.27.gate_proj", "model.layers.36.mlp.experts.28.gate_proj", "model.layers.36.mlp.experts.29.gate_proj", "model.layers.36.mlp.experts.30.gate_proj", "model.layers.36.mlp.experts.31.gate_proj", "model.layers.36.mlp.experts.32.gate_proj", "model.layers.36.mlp.experts.33.gate_proj", "model.layers.36.mlp.experts.34.gate_proj", "model.layers.36.mlp.experts.35.gate_proj", "model.layers.36.mlp.experts.36.gate_proj", "model.layers.36.mlp.experts.37.gate_proj", "model.layers.36.mlp.experts.38.gate_proj", "model.layers.36.mlp.experts.39.gate_proj", "model.layers.36.mlp.experts.40.gate_proj", "model.layers.36.mlp.experts.41.gate_proj", "model.layers.36.mlp.experts.42.gate_proj", "model.layers.36.mlp.experts.43.gate_proj", "model.layers.36.mlp.experts.44.gate_proj", "model.layers.36.mlp.experts.45.gate_proj", "model.layers.36.mlp.experts.46.gate_proj", "model.layers.36.mlp.experts.47.gate_proj", "model.layers.36.mlp.experts.48.gate_proj", "model.layers.36.mlp.experts.49.gate_proj", "model.layers.36.mlp.experts.50.gate_proj", "model.layers.36.mlp.experts.51.gate_proj", "model.layers.36.mlp.experts.52.gate_proj", "model.layers.36.mlp.experts.53.gate_proj", "model.layers.36.mlp.experts.54.gate_proj", "model.layers.36.mlp.experts.55.gate_proj", "model.layers.36.mlp.experts.56.gate_proj", "model.layers.36.mlp.experts.57.gate_proj", "model.layers.36.mlp.experts.58.gate_proj", "model.layers.36.mlp.experts.59.gate_proj", "model.layers.36.mlp.experts.60.gate_proj", "model.layers.36.mlp.experts.61.gate_proj", "model.layers.36.mlp.experts.62.gate_proj", "model.layers.36.mlp.experts.63.gate_proj", "model.layers.36.mlp.experts.64.gate_proj", "model.layers.36.mlp.experts.65.gate_proj", "model.layers.36.mlp.experts.66.gate_proj", "model.layers.36.mlp.experts.67.gate_proj", "model.layers.36.mlp.experts.68.gate_proj", "model.layers.36.mlp.experts.69.gate_proj", "model.layers.36.mlp.experts.70.gate_proj", "model.layers.36.mlp.experts.71.gate_proj", "model.layers.36.mlp.experts.72.gate_proj", "model.layers.36.mlp.experts.73.gate_proj", "model.layers.36.mlp.experts.74.gate_proj", "model.layers.36.mlp.experts.75.gate_proj", "model.layers.36.mlp.experts.76.gate_proj", "model.layers.36.mlp.experts.77.gate_proj", "model.layers.36.mlp.experts.78.gate_proj", "model.layers.36.mlp.experts.79.gate_proj", "model.layers.36.mlp.experts.80.gate_proj", "model.layers.36.mlp.experts.81.gate_proj", "model.layers.36.mlp.experts.82.gate_proj", "model.layers.36.mlp.experts.83.gate_proj", "model.layers.36.mlp.experts.84.gate_proj", "model.layers.36.mlp.experts.85.gate_proj", "model.layers.36.mlp.experts.86.gate_proj", "model.layers.36.mlp.experts.87.gate_proj", "model.layers.36.mlp.experts.88.gate_proj", "model.layers.36.mlp.experts.89.gate_proj", "model.layers.36.mlp.experts.90.gate_proj", "model.layers.36.mlp.experts.91.gate_proj", "model.layers.36.mlp.experts.92.gate_proj", "model.layers.36.mlp.experts.93.gate_proj", "model.layers.36.mlp.experts.94.gate_proj", "model.layers.36.mlp.experts.95.gate_proj", "model.layers.36.mlp.experts.96.gate_proj", "model.layers.36.mlp.experts.97.gate_proj", "model.layers.36.mlp.experts.98.gate_proj", "model.layers.36.mlp.experts.99.gate_proj", "model.layers.36.mlp.experts.100.gate_proj", "model.layers.36.mlp.experts.101.gate_proj", "model.layers.36.mlp.experts.102.gate_proj", "model.layers.36.mlp.experts.103.gate_proj", "model.layers.36.mlp.experts.104.gate_proj", "model.layers.36.mlp.experts.105.gate_proj", "model.layers.36.mlp.experts.106.gate_proj", "model.layers.36.mlp.experts.107.gate_proj", "model.layers.36.mlp.experts.108.gate_proj", "model.layers.36.mlp.experts.109.gate_proj", "model.layers.36.mlp.experts.110.gate_proj", "model.layers.36.mlp.experts.111.gate_proj", "model.layers.36.mlp.experts.112.gate_proj", "model.layers.36.mlp.experts.113.gate_proj", "model.layers.36.mlp.experts.114.gate_proj", "model.layers.36.mlp.experts.115.gate_proj", "model.layers.36.mlp.experts.116.gate_proj", "model.layers.36.mlp.experts.117.gate_proj", "model.layers.36.mlp.experts.118.gate_proj", "model.layers.36.mlp.experts.119.gate_proj", "model.layers.36.mlp.experts.120.gate_proj", "model.layers.36.mlp.experts.121.gate_proj", "model.layers.36.mlp.experts.122.gate_proj", "model.layers.36.mlp.experts.123.gate_proj", "model.layers.36.mlp.experts.124.gate_proj", "model.layers.36.mlp.experts.125.gate_proj", "model.layers.36.mlp.experts.126.gate_proj", "model.layers.36.mlp.experts.127.gate_proj", "model.layers.36.mlp.experts.128.gate_proj", "model.layers.36.mlp.experts.129.gate_proj", "model.layers.36.mlp.experts.130.gate_proj", "model.layers.36.mlp.experts.131.gate_proj", "model.layers.36.mlp.experts.132.gate_proj", "model.layers.36.mlp.experts.133.gate_proj", "model.layers.36.mlp.experts.134.gate_proj", "model.layers.36.mlp.experts.135.gate_proj", "model.layers.36.mlp.experts.136.gate_proj", "model.layers.36.mlp.experts.137.gate_proj", "model.layers.36.mlp.experts.138.gate_proj", "model.layers.36.mlp.experts.139.gate_proj", "model.layers.36.mlp.experts.140.gate_proj", "model.layers.36.mlp.experts.141.gate_proj", "model.layers.36.mlp.experts.142.gate_proj", "model.layers.36.mlp.experts.143.gate_proj", "model.layers.36.mlp.experts.144.gate_proj", "model.layers.36.mlp.experts.145.gate_proj", "model.layers.36.mlp.experts.146.gate_proj", "model.layers.36.mlp.experts.147.gate_proj", "model.layers.36.mlp.experts.148.gate_proj", "model.layers.36.mlp.experts.149.gate_proj", "model.layers.36.mlp.experts.150.gate_proj", "model.layers.36.mlp.experts.151.gate_proj", "model.layers.36.mlp.experts.152.gate_proj", "model.layers.36.mlp.experts.153.gate_proj", "model.layers.36.mlp.experts.154.gate_proj", "model.layers.36.mlp.experts.155.gate_proj", "model.layers.36.mlp.experts.156.gate_proj", "model.layers.36.mlp.experts.157.gate_proj", "model.layers.36.mlp.experts.158.gate_proj", "model.layers.36.mlp.experts.159.gate_proj", "model.layers.36.mlp.experts.0.up_proj", "model.layers.36.mlp.experts.1.up_proj", "model.layers.36.mlp.experts.2.up_proj", "model.layers.36.mlp.experts.3.up_proj", "model.layers.36.mlp.experts.4.up_proj", "model.layers.36.mlp.experts.5.up_proj", "model.layers.36.mlp.experts.6.up_proj", "model.layers.36.mlp.experts.7.up_proj", "model.layers.36.mlp.experts.8.up_proj", "model.layers.36.mlp.experts.9.up_proj", "model.layers.36.mlp.experts.10.up_proj", "model.layers.36.mlp.experts.11.up_proj", "model.layers.36.mlp.experts.12.up_proj", "model.layers.36.mlp.experts.13.up_proj", "model.layers.36.mlp.experts.14.up_proj", "model.layers.36.mlp.experts.15.up_proj", "model.layers.36.mlp.experts.16.up_proj", "model.layers.36.mlp.experts.17.up_proj", "model.layers.36.mlp.experts.18.up_proj", "model.layers.36.mlp.experts.19.up_proj", "model.layers.36.mlp.experts.20.up_proj", "model.layers.36.mlp.experts.21.up_proj", "model.layers.36.mlp.experts.22.up_proj", "model.layers.36.mlp.experts.23.up_proj", "model.layers.36.mlp.experts.24.up_proj", "model.layers.36.mlp.experts.25.up_proj", "model.layers.36.mlp.experts.26.up_proj", "model.layers.36.mlp.experts.27.up_proj", "model.layers.36.mlp.experts.28.up_proj", "model.layers.36.mlp.experts.29.up_proj", "model.layers.36.mlp.experts.30.up_proj", "model.layers.36.mlp.experts.31.up_proj", "model.layers.36.mlp.experts.32.up_proj", "model.layers.36.mlp.experts.33.up_proj", "model.layers.36.mlp.experts.34.up_proj", "model.layers.36.mlp.experts.35.up_proj", "model.layers.36.mlp.experts.36.up_proj", "model.layers.36.mlp.experts.37.up_proj", "model.layers.36.mlp.experts.38.up_proj", "model.layers.36.mlp.experts.39.up_proj", "model.layers.36.mlp.experts.40.up_proj", "model.layers.36.mlp.experts.41.up_proj", "model.layers.36.mlp.experts.42.up_proj", "model.layers.36.mlp.experts.43.up_proj", "model.layers.36.mlp.experts.44.up_proj", "model.layers.36.mlp.experts.45.up_proj", "model.layers.36.mlp.experts.46.up_proj", "model.layers.36.mlp.experts.47.up_proj", "model.layers.36.mlp.experts.48.up_proj", "model.layers.36.mlp.experts.49.up_proj", "model.layers.36.mlp.experts.50.up_proj", "model.layers.36.mlp.experts.51.up_proj", "model.layers.36.mlp.experts.52.up_proj", "model.layers.36.mlp.experts.53.up_proj", "model.layers.36.mlp.experts.54.up_proj", "model.layers.36.mlp.experts.55.up_proj", "model.layers.36.mlp.experts.56.up_proj", "model.layers.36.mlp.experts.57.up_proj", "model.layers.36.mlp.experts.58.up_proj", "model.layers.36.mlp.experts.59.up_proj", "model.layers.36.mlp.experts.60.up_proj", "model.layers.36.mlp.experts.61.up_proj", "model.layers.36.mlp.experts.62.up_proj", "model.layers.36.mlp.experts.63.up_proj", "model.layers.36.mlp.experts.64.up_proj", "model.layers.36.mlp.experts.65.up_proj", "model.layers.36.mlp.experts.66.up_proj", "model.layers.36.mlp.experts.67.up_proj", "model.layers.36.mlp.experts.68.up_proj", "model.layers.36.mlp.experts.69.up_proj", "model.layers.36.mlp.experts.70.up_proj", "model.layers.36.mlp.experts.71.up_proj", "model.layers.36.mlp.experts.72.up_proj", "model.layers.36.mlp.experts.73.up_proj", "model.layers.36.mlp.experts.74.up_proj", "model.layers.36.mlp.experts.75.up_proj", "model.layers.36.mlp.experts.76.up_proj", "model.layers.36.mlp.experts.77.up_proj", "model.layers.36.mlp.experts.78.up_proj", "model.layers.36.mlp.experts.79.up_proj", "model.layers.36.mlp.experts.80.up_proj", "model.layers.36.mlp.experts.81.up_proj", "model.layers.36.mlp.experts.82.up_proj", "model.layers.36.mlp.experts.83.up_proj", "model.layers.36.mlp.experts.84.up_proj", "model.layers.36.mlp.experts.85.up_proj", "model.layers.36.mlp.experts.86.up_proj", "model.layers.36.mlp.experts.87.up_proj", "model.layers.36.mlp.experts.88.up_proj", "model.layers.36.mlp.experts.89.up_proj", "model.layers.36.mlp.experts.90.up_proj", "model.layers.36.mlp.experts.91.up_proj", "model.layers.36.mlp.experts.92.up_proj", "model.layers.36.mlp.experts.93.up_proj", "model.layers.36.mlp.experts.94.up_proj", "model.layers.36.mlp.experts.95.up_proj", "model.layers.36.mlp.experts.96.up_proj", "model.layers.36.mlp.experts.97.up_proj", "model.layers.36.mlp.experts.98.up_proj", "model.layers.36.mlp.experts.99.up_proj", "model.layers.36.mlp.experts.100.up_proj", "model.layers.36.mlp.experts.101.up_proj", "model.layers.36.mlp.experts.102.up_proj", "model.layers.36.mlp.experts.103.up_proj", "model.layers.36.mlp.experts.104.up_proj", "model.layers.36.mlp.experts.105.up_proj", "model.layers.36.mlp.experts.106.up_proj", "model.layers.36.mlp.experts.107.up_proj", "model.layers.36.mlp.experts.108.up_proj", "model.layers.36.mlp.experts.109.up_proj", "model.layers.36.mlp.experts.110.up_proj", "model.layers.36.mlp.experts.111.up_proj", "model.layers.36.mlp.experts.112.up_proj", "model.layers.36.mlp.experts.113.up_proj", "model.layers.36.mlp.experts.114.up_proj", "model.layers.36.mlp.experts.115.up_proj", "model.layers.36.mlp.experts.116.up_proj", "model.layers.36.mlp.experts.117.up_proj", "model.layers.36.mlp.experts.118.up_proj", "model.layers.36.mlp.experts.119.up_proj", "model.layers.36.mlp.experts.120.up_proj", "model.layers.36.mlp.experts.121.up_proj", "model.layers.36.mlp.experts.122.up_proj", "model.layers.36.mlp.experts.123.up_proj", "model.layers.36.mlp.experts.124.up_proj", "model.layers.36.mlp.experts.125.up_proj", "model.layers.36.mlp.experts.126.up_proj", "model.layers.36.mlp.experts.127.up_proj", "model.layers.36.mlp.experts.128.up_proj", "model.layers.36.mlp.experts.129.up_proj", "model.layers.36.mlp.experts.130.up_proj", "model.layers.36.mlp.experts.131.up_proj", "model.layers.36.mlp.experts.132.up_proj", "model.layers.36.mlp.experts.133.up_proj", "model.layers.36.mlp.experts.134.up_proj", "model.layers.36.mlp.experts.135.up_proj", "model.layers.36.mlp.experts.136.up_proj", "model.layers.36.mlp.experts.137.up_proj", "model.layers.36.mlp.experts.138.up_proj", "model.layers.36.mlp.experts.139.up_proj", "model.layers.36.mlp.experts.140.up_proj", "model.layers.36.mlp.experts.141.up_proj", "model.layers.36.mlp.experts.142.up_proj", "model.layers.36.mlp.experts.143.up_proj", "model.layers.36.mlp.experts.144.up_proj", "model.layers.36.mlp.experts.145.up_proj", "model.layers.36.mlp.experts.146.up_proj", "model.layers.36.mlp.experts.147.up_proj", "model.layers.36.mlp.experts.148.up_proj", "model.layers.36.mlp.experts.149.up_proj", "model.layers.36.mlp.experts.150.up_proj", "model.layers.36.mlp.experts.151.up_proj", "model.layers.36.mlp.experts.152.up_proj", "model.layers.36.mlp.experts.153.up_proj", "model.layers.36.mlp.experts.154.up_proj", "model.layers.36.mlp.experts.155.up_proj", "model.layers.36.mlp.experts.156.up_proj", "model.layers.36.mlp.experts.157.up_proj", "model.layers.36.mlp.experts.158.up_proj", "model.layers.36.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.9133789464831135e-06, "dbits": 5033164800 } ] }, { "idx": 218, "layers": [ "model.layers.36.mlp.experts.0.down_proj", "model.layers.36.mlp.experts.1.down_proj", "model.layers.36.mlp.experts.2.down_proj", "model.layers.36.mlp.experts.3.down_proj", "model.layers.36.mlp.experts.4.down_proj", "model.layers.36.mlp.experts.5.down_proj", "model.layers.36.mlp.experts.6.down_proj", "model.layers.36.mlp.experts.7.down_proj", "model.layers.36.mlp.experts.8.down_proj", "model.layers.36.mlp.experts.9.down_proj", "model.layers.36.mlp.experts.10.down_proj", "model.layers.36.mlp.experts.11.down_proj", "model.layers.36.mlp.experts.12.down_proj", "model.layers.36.mlp.experts.13.down_proj", "model.layers.36.mlp.experts.14.down_proj", "model.layers.36.mlp.experts.15.down_proj", "model.layers.36.mlp.experts.16.down_proj", "model.layers.36.mlp.experts.17.down_proj", "model.layers.36.mlp.experts.18.down_proj", "model.layers.36.mlp.experts.19.down_proj", "model.layers.36.mlp.experts.20.down_proj", "model.layers.36.mlp.experts.21.down_proj", "model.layers.36.mlp.experts.22.down_proj", "model.layers.36.mlp.experts.23.down_proj", "model.layers.36.mlp.experts.24.down_proj", "model.layers.36.mlp.experts.25.down_proj", "model.layers.36.mlp.experts.26.down_proj", "model.layers.36.mlp.experts.27.down_proj", "model.layers.36.mlp.experts.28.down_proj", "model.layers.36.mlp.experts.29.down_proj", "model.layers.36.mlp.experts.30.down_proj", "model.layers.36.mlp.experts.31.down_proj", "model.layers.36.mlp.experts.32.down_proj", "model.layers.36.mlp.experts.33.down_proj", "model.layers.36.mlp.experts.34.down_proj", "model.layers.36.mlp.experts.35.down_proj", "model.layers.36.mlp.experts.36.down_proj", "model.layers.36.mlp.experts.37.down_proj", "model.layers.36.mlp.experts.38.down_proj", "model.layers.36.mlp.experts.39.down_proj", "model.layers.36.mlp.experts.40.down_proj", "model.layers.36.mlp.experts.41.down_proj", "model.layers.36.mlp.experts.42.down_proj", "model.layers.36.mlp.experts.43.down_proj", "model.layers.36.mlp.experts.44.down_proj", "model.layers.36.mlp.experts.45.down_proj", "model.layers.36.mlp.experts.46.down_proj", "model.layers.36.mlp.experts.47.down_proj", "model.layers.36.mlp.experts.48.down_proj", "model.layers.36.mlp.experts.49.down_proj", "model.layers.36.mlp.experts.50.down_proj", "model.layers.36.mlp.experts.51.down_proj", "model.layers.36.mlp.experts.52.down_proj", "model.layers.36.mlp.experts.53.down_proj", "model.layers.36.mlp.experts.54.down_proj", "model.layers.36.mlp.experts.55.down_proj", "model.layers.36.mlp.experts.56.down_proj", "model.layers.36.mlp.experts.57.down_proj", "model.layers.36.mlp.experts.58.down_proj", "model.layers.36.mlp.experts.59.down_proj", "model.layers.36.mlp.experts.60.down_proj", "model.layers.36.mlp.experts.61.down_proj", "model.layers.36.mlp.experts.62.down_proj", "model.layers.36.mlp.experts.63.down_proj", "model.layers.36.mlp.experts.64.down_proj", "model.layers.36.mlp.experts.65.down_proj", "model.layers.36.mlp.experts.66.down_proj", "model.layers.36.mlp.experts.67.down_proj", "model.layers.36.mlp.experts.68.down_proj", "model.layers.36.mlp.experts.69.down_proj", "model.layers.36.mlp.experts.70.down_proj", "model.layers.36.mlp.experts.71.down_proj", "model.layers.36.mlp.experts.72.down_proj", "model.layers.36.mlp.experts.73.down_proj", "model.layers.36.mlp.experts.74.down_proj", "model.layers.36.mlp.experts.75.down_proj", "model.layers.36.mlp.experts.76.down_proj", "model.layers.36.mlp.experts.77.down_proj", "model.layers.36.mlp.experts.78.down_proj", "model.layers.36.mlp.experts.79.down_proj", "model.layers.36.mlp.experts.80.down_proj", "model.layers.36.mlp.experts.81.down_proj", "model.layers.36.mlp.experts.82.down_proj", "model.layers.36.mlp.experts.83.down_proj", "model.layers.36.mlp.experts.84.down_proj", "model.layers.36.mlp.experts.85.down_proj", "model.layers.36.mlp.experts.86.down_proj", "model.layers.36.mlp.experts.87.down_proj", "model.layers.36.mlp.experts.88.down_proj", "model.layers.36.mlp.experts.89.down_proj", "model.layers.36.mlp.experts.90.down_proj", "model.layers.36.mlp.experts.91.down_proj", "model.layers.36.mlp.experts.92.down_proj", "model.layers.36.mlp.experts.93.down_proj", "model.layers.36.mlp.experts.94.down_proj", "model.layers.36.mlp.experts.95.down_proj", "model.layers.36.mlp.experts.96.down_proj", "model.layers.36.mlp.experts.97.down_proj", "model.layers.36.mlp.experts.98.down_proj", "model.layers.36.mlp.experts.99.down_proj", "model.layers.36.mlp.experts.100.down_proj", "model.layers.36.mlp.experts.101.down_proj", "model.layers.36.mlp.experts.102.down_proj", "model.layers.36.mlp.experts.103.down_proj", "model.layers.36.mlp.experts.104.down_proj", "model.layers.36.mlp.experts.105.down_proj", "model.layers.36.mlp.experts.106.down_proj", "model.layers.36.mlp.experts.107.down_proj", "model.layers.36.mlp.experts.108.down_proj", "model.layers.36.mlp.experts.109.down_proj", "model.layers.36.mlp.experts.110.down_proj", "model.layers.36.mlp.experts.111.down_proj", "model.layers.36.mlp.experts.112.down_proj", "model.layers.36.mlp.experts.113.down_proj", "model.layers.36.mlp.experts.114.down_proj", "model.layers.36.mlp.experts.115.down_proj", "model.layers.36.mlp.experts.116.down_proj", "model.layers.36.mlp.experts.117.down_proj", "model.layers.36.mlp.experts.118.down_proj", "model.layers.36.mlp.experts.119.down_proj", "model.layers.36.mlp.experts.120.down_proj", "model.layers.36.mlp.experts.121.down_proj", "model.layers.36.mlp.experts.122.down_proj", "model.layers.36.mlp.experts.123.down_proj", "model.layers.36.mlp.experts.124.down_proj", "model.layers.36.mlp.experts.125.down_proj", "model.layers.36.mlp.experts.126.down_proj", "model.layers.36.mlp.experts.127.down_proj", "model.layers.36.mlp.experts.128.down_proj", "model.layers.36.mlp.experts.129.down_proj", "model.layers.36.mlp.experts.130.down_proj", "model.layers.36.mlp.experts.131.down_proj", "model.layers.36.mlp.experts.132.down_proj", "model.layers.36.mlp.experts.133.down_proj", "model.layers.36.mlp.experts.134.down_proj", "model.layers.36.mlp.experts.135.down_proj", "model.layers.36.mlp.experts.136.down_proj", "model.layers.36.mlp.experts.137.down_proj", "model.layers.36.mlp.experts.138.down_proj", "model.layers.36.mlp.experts.139.down_proj", "model.layers.36.mlp.experts.140.down_proj", "model.layers.36.mlp.experts.141.down_proj", "model.layers.36.mlp.experts.142.down_proj", "model.layers.36.mlp.experts.143.down_proj", "model.layers.36.mlp.experts.144.down_proj", "model.layers.36.mlp.experts.145.down_proj", "model.layers.36.mlp.experts.146.down_proj", "model.layers.36.mlp.experts.147.down_proj", "model.layers.36.mlp.experts.148.down_proj", "model.layers.36.mlp.experts.149.down_proj", "model.layers.36.mlp.experts.150.down_proj", "model.layers.36.mlp.experts.151.down_proj", "model.layers.36.mlp.experts.152.down_proj", "model.layers.36.mlp.experts.153.down_proj", "model.layers.36.mlp.experts.154.down_proj", "model.layers.36.mlp.experts.155.down_proj", "model.layers.36.mlp.experts.156.down_proj", "model.layers.36.mlp.experts.157.down_proj", "model.layers.36.mlp.experts.158.down_proj", "model.layers.36.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -2.771703293547034e-07, "dbits": 2516582400 } ] }, { "idx": 219, "layers": [ "model.layers.37.self_attn.q_proj" ], "candidates": [ { "dkld": -6.2643492128699584e-06, "dbits": 125829120 } ] }, { "idx": 220, "layers": [ "model.layers.37.self_attn.k_proj", "model.layers.37.self_attn.v_proj" ], "candidates": [ { "dkld": -1.1648656800389312e-05, "dbits": 20971520 } ] }, { "idx": 221, "layers": [ "model.layers.37.self_attn.o_proj" ], "candidates": [ { "dkld": 2.6505615096539693e-06, "dbits": 125829120 } ] }, { "idx": 222, "layers": [ "model.layers.37.mlp.shared_experts.gate_proj", "model.layers.37.mlp.shared_experts.up_proj", "model.layers.37.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 9.659095667302175e-07, "dbits": 47185920 } ] }, { "idx": 223, "layers": [ "model.layers.37.mlp.experts.0.gate_proj", "model.layers.37.mlp.experts.1.gate_proj", "model.layers.37.mlp.experts.2.gate_proj", "model.layers.37.mlp.experts.3.gate_proj", "model.layers.37.mlp.experts.4.gate_proj", "model.layers.37.mlp.experts.5.gate_proj", "model.layers.37.mlp.experts.6.gate_proj", "model.layers.37.mlp.experts.7.gate_proj", "model.layers.37.mlp.experts.8.gate_proj", "model.layers.37.mlp.experts.9.gate_proj", "model.layers.37.mlp.experts.10.gate_proj", "model.layers.37.mlp.experts.11.gate_proj", "model.layers.37.mlp.experts.12.gate_proj", "model.layers.37.mlp.experts.13.gate_proj", "model.layers.37.mlp.experts.14.gate_proj", "model.layers.37.mlp.experts.15.gate_proj", "model.layers.37.mlp.experts.16.gate_proj", "model.layers.37.mlp.experts.17.gate_proj", "model.layers.37.mlp.experts.18.gate_proj", "model.layers.37.mlp.experts.19.gate_proj", "model.layers.37.mlp.experts.20.gate_proj", "model.layers.37.mlp.experts.21.gate_proj", "model.layers.37.mlp.experts.22.gate_proj", "model.layers.37.mlp.experts.23.gate_proj", "model.layers.37.mlp.experts.24.gate_proj", "model.layers.37.mlp.experts.25.gate_proj", "model.layers.37.mlp.experts.26.gate_proj", "model.layers.37.mlp.experts.27.gate_proj", "model.layers.37.mlp.experts.28.gate_proj", "model.layers.37.mlp.experts.29.gate_proj", "model.layers.37.mlp.experts.30.gate_proj", "model.layers.37.mlp.experts.31.gate_proj", "model.layers.37.mlp.experts.32.gate_proj", "model.layers.37.mlp.experts.33.gate_proj", "model.layers.37.mlp.experts.34.gate_proj", "model.layers.37.mlp.experts.35.gate_proj", "model.layers.37.mlp.experts.36.gate_proj", "model.layers.37.mlp.experts.37.gate_proj", "model.layers.37.mlp.experts.38.gate_proj", "model.layers.37.mlp.experts.39.gate_proj", "model.layers.37.mlp.experts.40.gate_proj", "model.layers.37.mlp.experts.41.gate_proj", "model.layers.37.mlp.experts.42.gate_proj", "model.layers.37.mlp.experts.43.gate_proj", "model.layers.37.mlp.experts.44.gate_proj", "model.layers.37.mlp.experts.45.gate_proj", "model.layers.37.mlp.experts.46.gate_proj", "model.layers.37.mlp.experts.47.gate_proj", "model.layers.37.mlp.experts.48.gate_proj", "model.layers.37.mlp.experts.49.gate_proj", "model.layers.37.mlp.experts.50.gate_proj", "model.layers.37.mlp.experts.51.gate_proj", "model.layers.37.mlp.experts.52.gate_proj", "model.layers.37.mlp.experts.53.gate_proj", "model.layers.37.mlp.experts.54.gate_proj", "model.layers.37.mlp.experts.55.gate_proj", "model.layers.37.mlp.experts.56.gate_proj", "model.layers.37.mlp.experts.57.gate_proj", "model.layers.37.mlp.experts.58.gate_proj", "model.layers.37.mlp.experts.59.gate_proj", "model.layers.37.mlp.experts.60.gate_proj", "model.layers.37.mlp.experts.61.gate_proj", "model.layers.37.mlp.experts.62.gate_proj", "model.layers.37.mlp.experts.63.gate_proj", "model.layers.37.mlp.experts.64.gate_proj", "model.layers.37.mlp.experts.65.gate_proj", "model.layers.37.mlp.experts.66.gate_proj", "model.layers.37.mlp.experts.67.gate_proj", "model.layers.37.mlp.experts.68.gate_proj", "model.layers.37.mlp.experts.69.gate_proj", "model.layers.37.mlp.experts.70.gate_proj", "model.layers.37.mlp.experts.71.gate_proj", "model.layers.37.mlp.experts.72.gate_proj", "model.layers.37.mlp.experts.73.gate_proj", "model.layers.37.mlp.experts.74.gate_proj", "model.layers.37.mlp.experts.75.gate_proj", "model.layers.37.mlp.experts.76.gate_proj", "model.layers.37.mlp.experts.77.gate_proj", "model.layers.37.mlp.experts.78.gate_proj", "model.layers.37.mlp.experts.79.gate_proj", "model.layers.37.mlp.experts.80.gate_proj", "model.layers.37.mlp.experts.81.gate_proj", "model.layers.37.mlp.experts.82.gate_proj", "model.layers.37.mlp.experts.83.gate_proj", "model.layers.37.mlp.experts.84.gate_proj", "model.layers.37.mlp.experts.85.gate_proj", "model.layers.37.mlp.experts.86.gate_proj", "model.layers.37.mlp.experts.87.gate_proj", "model.layers.37.mlp.experts.88.gate_proj", "model.layers.37.mlp.experts.89.gate_proj", "model.layers.37.mlp.experts.90.gate_proj", "model.layers.37.mlp.experts.91.gate_proj", "model.layers.37.mlp.experts.92.gate_proj", "model.layers.37.mlp.experts.93.gate_proj", "model.layers.37.mlp.experts.94.gate_proj", "model.layers.37.mlp.experts.95.gate_proj", "model.layers.37.mlp.experts.96.gate_proj", "model.layers.37.mlp.experts.97.gate_proj", "model.layers.37.mlp.experts.98.gate_proj", "model.layers.37.mlp.experts.99.gate_proj", "model.layers.37.mlp.experts.100.gate_proj", "model.layers.37.mlp.experts.101.gate_proj", "model.layers.37.mlp.experts.102.gate_proj", "model.layers.37.mlp.experts.103.gate_proj", "model.layers.37.mlp.experts.104.gate_proj", "model.layers.37.mlp.experts.105.gate_proj", "model.layers.37.mlp.experts.106.gate_proj", "model.layers.37.mlp.experts.107.gate_proj", "model.layers.37.mlp.experts.108.gate_proj", "model.layers.37.mlp.experts.109.gate_proj", "model.layers.37.mlp.experts.110.gate_proj", "model.layers.37.mlp.experts.111.gate_proj", "model.layers.37.mlp.experts.112.gate_proj", "model.layers.37.mlp.experts.113.gate_proj", "model.layers.37.mlp.experts.114.gate_proj", "model.layers.37.mlp.experts.115.gate_proj", "model.layers.37.mlp.experts.116.gate_proj", "model.layers.37.mlp.experts.117.gate_proj", "model.layers.37.mlp.experts.118.gate_proj", "model.layers.37.mlp.experts.119.gate_proj", "model.layers.37.mlp.experts.120.gate_proj", "model.layers.37.mlp.experts.121.gate_proj", "model.layers.37.mlp.experts.122.gate_proj", "model.layers.37.mlp.experts.123.gate_proj", "model.layers.37.mlp.experts.124.gate_proj", "model.layers.37.mlp.experts.125.gate_proj", "model.layers.37.mlp.experts.126.gate_proj", "model.layers.37.mlp.experts.127.gate_proj", "model.layers.37.mlp.experts.128.gate_proj", "model.layers.37.mlp.experts.129.gate_proj", "model.layers.37.mlp.experts.130.gate_proj", "model.layers.37.mlp.experts.131.gate_proj", "model.layers.37.mlp.experts.132.gate_proj", "model.layers.37.mlp.experts.133.gate_proj", "model.layers.37.mlp.experts.134.gate_proj", "model.layers.37.mlp.experts.135.gate_proj", "model.layers.37.mlp.experts.136.gate_proj", "model.layers.37.mlp.experts.137.gate_proj", "model.layers.37.mlp.experts.138.gate_proj", "model.layers.37.mlp.experts.139.gate_proj", "model.layers.37.mlp.experts.140.gate_proj", "model.layers.37.mlp.experts.141.gate_proj", "model.layers.37.mlp.experts.142.gate_proj", "model.layers.37.mlp.experts.143.gate_proj", "model.layers.37.mlp.experts.144.gate_proj", "model.layers.37.mlp.experts.145.gate_proj", "model.layers.37.mlp.experts.146.gate_proj", "model.layers.37.mlp.experts.147.gate_proj", "model.layers.37.mlp.experts.148.gate_proj", "model.layers.37.mlp.experts.149.gate_proj", "model.layers.37.mlp.experts.150.gate_proj", "model.layers.37.mlp.experts.151.gate_proj", "model.layers.37.mlp.experts.152.gate_proj", "model.layers.37.mlp.experts.153.gate_proj", "model.layers.37.mlp.experts.154.gate_proj", "model.layers.37.mlp.experts.155.gate_proj", "model.layers.37.mlp.experts.156.gate_proj", "model.layers.37.mlp.experts.157.gate_proj", "model.layers.37.mlp.experts.158.gate_proj", "model.layers.37.mlp.experts.159.gate_proj", "model.layers.37.mlp.experts.0.up_proj", "model.layers.37.mlp.experts.1.up_proj", "model.layers.37.mlp.experts.2.up_proj", "model.layers.37.mlp.experts.3.up_proj", "model.layers.37.mlp.experts.4.up_proj", "model.layers.37.mlp.experts.5.up_proj", "model.layers.37.mlp.experts.6.up_proj", "model.layers.37.mlp.experts.7.up_proj", "model.layers.37.mlp.experts.8.up_proj", "model.layers.37.mlp.experts.9.up_proj", "model.layers.37.mlp.experts.10.up_proj", "model.layers.37.mlp.experts.11.up_proj", "model.layers.37.mlp.experts.12.up_proj", "model.layers.37.mlp.experts.13.up_proj", "model.layers.37.mlp.experts.14.up_proj", "model.layers.37.mlp.experts.15.up_proj", "model.layers.37.mlp.experts.16.up_proj", "model.layers.37.mlp.experts.17.up_proj", "model.layers.37.mlp.experts.18.up_proj", "model.layers.37.mlp.experts.19.up_proj", "model.layers.37.mlp.experts.20.up_proj", "model.layers.37.mlp.experts.21.up_proj", "model.layers.37.mlp.experts.22.up_proj", "model.layers.37.mlp.experts.23.up_proj", "model.layers.37.mlp.experts.24.up_proj", "model.layers.37.mlp.experts.25.up_proj", "model.layers.37.mlp.experts.26.up_proj", "model.layers.37.mlp.experts.27.up_proj", "model.layers.37.mlp.experts.28.up_proj", "model.layers.37.mlp.experts.29.up_proj", "model.layers.37.mlp.experts.30.up_proj", "model.layers.37.mlp.experts.31.up_proj", "model.layers.37.mlp.experts.32.up_proj", "model.layers.37.mlp.experts.33.up_proj", "model.layers.37.mlp.experts.34.up_proj", "model.layers.37.mlp.experts.35.up_proj", "model.layers.37.mlp.experts.36.up_proj", "model.layers.37.mlp.experts.37.up_proj", "model.layers.37.mlp.experts.38.up_proj", "model.layers.37.mlp.experts.39.up_proj", "model.layers.37.mlp.experts.40.up_proj", "model.layers.37.mlp.experts.41.up_proj", "model.layers.37.mlp.experts.42.up_proj", "model.layers.37.mlp.experts.43.up_proj", "model.layers.37.mlp.experts.44.up_proj", "model.layers.37.mlp.experts.45.up_proj", "model.layers.37.mlp.experts.46.up_proj", "model.layers.37.mlp.experts.47.up_proj", "model.layers.37.mlp.experts.48.up_proj", "model.layers.37.mlp.experts.49.up_proj", "model.layers.37.mlp.experts.50.up_proj", "model.layers.37.mlp.experts.51.up_proj", "model.layers.37.mlp.experts.52.up_proj", "model.layers.37.mlp.experts.53.up_proj", "model.layers.37.mlp.experts.54.up_proj", "model.layers.37.mlp.experts.55.up_proj", "model.layers.37.mlp.experts.56.up_proj", "model.layers.37.mlp.experts.57.up_proj", "model.layers.37.mlp.experts.58.up_proj", "model.layers.37.mlp.experts.59.up_proj", "model.layers.37.mlp.experts.60.up_proj", "model.layers.37.mlp.experts.61.up_proj", "model.layers.37.mlp.experts.62.up_proj", "model.layers.37.mlp.experts.63.up_proj", "model.layers.37.mlp.experts.64.up_proj", "model.layers.37.mlp.experts.65.up_proj", "model.layers.37.mlp.experts.66.up_proj", "model.layers.37.mlp.experts.67.up_proj", "model.layers.37.mlp.experts.68.up_proj", "model.layers.37.mlp.experts.69.up_proj", "model.layers.37.mlp.experts.70.up_proj", "model.layers.37.mlp.experts.71.up_proj", "model.layers.37.mlp.experts.72.up_proj", "model.layers.37.mlp.experts.73.up_proj", "model.layers.37.mlp.experts.74.up_proj", "model.layers.37.mlp.experts.75.up_proj", "model.layers.37.mlp.experts.76.up_proj", "model.layers.37.mlp.experts.77.up_proj", "model.layers.37.mlp.experts.78.up_proj", "model.layers.37.mlp.experts.79.up_proj", "model.layers.37.mlp.experts.80.up_proj", "model.layers.37.mlp.experts.81.up_proj", "model.layers.37.mlp.experts.82.up_proj", "model.layers.37.mlp.experts.83.up_proj", "model.layers.37.mlp.experts.84.up_proj", "model.layers.37.mlp.experts.85.up_proj", "model.layers.37.mlp.experts.86.up_proj", "model.layers.37.mlp.experts.87.up_proj", "model.layers.37.mlp.experts.88.up_proj", "model.layers.37.mlp.experts.89.up_proj", "model.layers.37.mlp.experts.90.up_proj", "model.layers.37.mlp.experts.91.up_proj", "model.layers.37.mlp.experts.92.up_proj", "model.layers.37.mlp.experts.93.up_proj", "model.layers.37.mlp.experts.94.up_proj", "model.layers.37.mlp.experts.95.up_proj", "model.layers.37.mlp.experts.96.up_proj", "model.layers.37.mlp.experts.97.up_proj", "model.layers.37.mlp.experts.98.up_proj", "model.layers.37.mlp.experts.99.up_proj", "model.layers.37.mlp.experts.100.up_proj", "model.layers.37.mlp.experts.101.up_proj", "model.layers.37.mlp.experts.102.up_proj", "model.layers.37.mlp.experts.103.up_proj", "model.layers.37.mlp.experts.104.up_proj", "model.layers.37.mlp.experts.105.up_proj", "model.layers.37.mlp.experts.106.up_proj", "model.layers.37.mlp.experts.107.up_proj", "model.layers.37.mlp.experts.108.up_proj", "model.layers.37.mlp.experts.109.up_proj", "model.layers.37.mlp.experts.110.up_proj", "model.layers.37.mlp.experts.111.up_proj", "model.layers.37.mlp.experts.112.up_proj", "model.layers.37.mlp.experts.113.up_proj", "model.layers.37.mlp.experts.114.up_proj", "model.layers.37.mlp.experts.115.up_proj", "model.layers.37.mlp.experts.116.up_proj", "model.layers.37.mlp.experts.117.up_proj", "model.layers.37.mlp.experts.118.up_proj", "model.layers.37.mlp.experts.119.up_proj", "model.layers.37.mlp.experts.120.up_proj", "model.layers.37.mlp.experts.121.up_proj", "model.layers.37.mlp.experts.122.up_proj", "model.layers.37.mlp.experts.123.up_proj", "model.layers.37.mlp.experts.124.up_proj", "model.layers.37.mlp.experts.125.up_proj", "model.layers.37.mlp.experts.126.up_proj", "model.layers.37.mlp.experts.127.up_proj", "model.layers.37.mlp.experts.128.up_proj", "model.layers.37.mlp.experts.129.up_proj", "model.layers.37.mlp.experts.130.up_proj", "model.layers.37.mlp.experts.131.up_proj", "model.layers.37.mlp.experts.132.up_proj", "model.layers.37.mlp.experts.133.up_proj", "model.layers.37.mlp.experts.134.up_proj", "model.layers.37.mlp.experts.135.up_proj", "model.layers.37.mlp.experts.136.up_proj", "model.layers.37.mlp.experts.137.up_proj", "model.layers.37.mlp.experts.138.up_proj", "model.layers.37.mlp.experts.139.up_proj", "model.layers.37.mlp.experts.140.up_proj", "model.layers.37.mlp.experts.141.up_proj", "model.layers.37.mlp.experts.142.up_proj", "model.layers.37.mlp.experts.143.up_proj", "model.layers.37.mlp.experts.144.up_proj", "model.layers.37.mlp.experts.145.up_proj", "model.layers.37.mlp.experts.146.up_proj", "model.layers.37.mlp.experts.147.up_proj", "model.layers.37.mlp.experts.148.up_proj", "model.layers.37.mlp.experts.149.up_proj", "model.layers.37.mlp.experts.150.up_proj", "model.layers.37.mlp.experts.151.up_proj", "model.layers.37.mlp.experts.152.up_proj", "model.layers.37.mlp.experts.153.up_proj", "model.layers.37.mlp.experts.154.up_proj", "model.layers.37.mlp.experts.155.up_proj", "model.layers.37.mlp.experts.156.up_proj", "model.layers.37.mlp.experts.157.up_proj", "model.layers.37.mlp.experts.158.up_proj", "model.layers.37.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.3332464732273078e-08, "dbits": 5033164800 } ] }, { "idx": 224, "layers": [ "model.layers.37.mlp.experts.0.down_proj", "model.layers.37.mlp.experts.1.down_proj", "model.layers.37.mlp.experts.2.down_proj", "model.layers.37.mlp.experts.3.down_proj", "model.layers.37.mlp.experts.4.down_proj", "model.layers.37.mlp.experts.5.down_proj", "model.layers.37.mlp.experts.6.down_proj", "model.layers.37.mlp.experts.7.down_proj", "model.layers.37.mlp.experts.8.down_proj", "model.layers.37.mlp.experts.9.down_proj", "model.layers.37.mlp.experts.10.down_proj", "model.layers.37.mlp.experts.11.down_proj", "model.layers.37.mlp.experts.12.down_proj", "model.layers.37.mlp.experts.13.down_proj", "model.layers.37.mlp.experts.14.down_proj", "model.layers.37.mlp.experts.15.down_proj", "model.layers.37.mlp.experts.16.down_proj", "model.layers.37.mlp.experts.17.down_proj", "model.layers.37.mlp.experts.18.down_proj", "model.layers.37.mlp.experts.19.down_proj", "model.layers.37.mlp.experts.20.down_proj", "model.layers.37.mlp.experts.21.down_proj", "model.layers.37.mlp.experts.22.down_proj", "model.layers.37.mlp.experts.23.down_proj", "model.layers.37.mlp.experts.24.down_proj", "model.layers.37.mlp.experts.25.down_proj", "model.layers.37.mlp.experts.26.down_proj", "model.layers.37.mlp.experts.27.down_proj", "model.layers.37.mlp.experts.28.down_proj", "model.layers.37.mlp.experts.29.down_proj", "model.layers.37.mlp.experts.30.down_proj", "model.layers.37.mlp.experts.31.down_proj", "model.layers.37.mlp.experts.32.down_proj", "model.layers.37.mlp.experts.33.down_proj", "model.layers.37.mlp.experts.34.down_proj", "model.layers.37.mlp.experts.35.down_proj", "model.layers.37.mlp.experts.36.down_proj", "model.layers.37.mlp.experts.37.down_proj", "model.layers.37.mlp.experts.38.down_proj", "model.layers.37.mlp.experts.39.down_proj", "model.layers.37.mlp.experts.40.down_proj", "model.layers.37.mlp.experts.41.down_proj", "model.layers.37.mlp.experts.42.down_proj", "model.layers.37.mlp.experts.43.down_proj", "model.layers.37.mlp.experts.44.down_proj", "model.layers.37.mlp.experts.45.down_proj", "model.layers.37.mlp.experts.46.down_proj", "model.layers.37.mlp.experts.47.down_proj", "model.layers.37.mlp.experts.48.down_proj", "model.layers.37.mlp.experts.49.down_proj", "model.layers.37.mlp.experts.50.down_proj", "model.layers.37.mlp.experts.51.down_proj", "model.layers.37.mlp.experts.52.down_proj", "model.layers.37.mlp.experts.53.down_proj", "model.layers.37.mlp.experts.54.down_proj", "model.layers.37.mlp.experts.55.down_proj", "model.layers.37.mlp.experts.56.down_proj", "model.layers.37.mlp.experts.57.down_proj", "model.layers.37.mlp.experts.58.down_proj", "model.layers.37.mlp.experts.59.down_proj", "model.layers.37.mlp.experts.60.down_proj", "model.layers.37.mlp.experts.61.down_proj", "model.layers.37.mlp.experts.62.down_proj", "model.layers.37.mlp.experts.63.down_proj", "model.layers.37.mlp.experts.64.down_proj", "model.layers.37.mlp.experts.65.down_proj", "model.layers.37.mlp.experts.66.down_proj", "model.layers.37.mlp.experts.67.down_proj", "model.layers.37.mlp.experts.68.down_proj", "model.layers.37.mlp.experts.69.down_proj", "model.layers.37.mlp.experts.70.down_proj", "model.layers.37.mlp.experts.71.down_proj", "model.layers.37.mlp.experts.72.down_proj", "model.layers.37.mlp.experts.73.down_proj", "model.layers.37.mlp.experts.74.down_proj", "model.layers.37.mlp.experts.75.down_proj", "model.layers.37.mlp.experts.76.down_proj", "model.layers.37.mlp.experts.77.down_proj", "model.layers.37.mlp.experts.78.down_proj", "model.layers.37.mlp.experts.79.down_proj", "model.layers.37.mlp.experts.80.down_proj", "model.layers.37.mlp.experts.81.down_proj", "model.layers.37.mlp.experts.82.down_proj", "model.layers.37.mlp.experts.83.down_proj", "model.layers.37.mlp.experts.84.down_proj", "model.layers.37.mlp.experts.85.down_proj", "model.layers.37.mlp.experts.86.down_proj", "model.layers.37.mlp.experts.87.down_proj", "model.layers.37.mlp.experts.88.down_proj", "model.layers.37.mlp.experts.89.down_proj", "model.layers.37.mlp.experts.90.down_proj", "model.layers.37.mlp.experts.91.down_proj", "model.layers.37.mlp.experts.92.down_proj", "model.layers.37.mlp.experts.93.down_proj", "model.layers.37.mlp.experts.94.down_proj", "model.layers.37.mlp.experts.95.down_proj", "model.layers.37.mlp.experts.96.down_proj", "model.layers.37.mlp.experts.97.down_proj", "model.layers.37.mlp.experts.98.down_proj", "model.layers.37.mlp.experts.99.down_proj", "model.layers.37.mlp.experts.100.down_proj", "model.layers.37.mlp.experts.101.down_proj", "model.layers.37.mlp.experts.102.down_proj", "model.layers.37.mlp.experts.103.down_proj", "model.layers.37.mlp.experts.104.down_proj", "model.layers.37.mlp.experts.105.down_proj", "model.layers.37.mlp.experts.106.down_proj", "model.layers.37.mlp.experts.107.down_proj", "model.layers.37.mlp.experts.108.down_proj", "model.layers.37.mlp.experts.109.down_proj", "model.layers.37.mlp.experts.110.down_proj", "model.layers.37.mlp.experts.111.down_proj", "model.layers.37.mlp.experts.112.down_proj", "model.layers.37.mlp.experts.113.down_proj", "model.layers.37.mlp.experts.114.down_proj", "model.layers.37.mlp.experts.115.down_proj", "model.layers.37.mlp.experts.116.down_proj", "model.layers.37.mlp.experts.117.down_proj", "model.layers.37.mlp.experts.118.down_proj", "model.layers.37.mlp.experts.119.down_proj", "model.layers.37.mlp.experts.120.down_proj", "model.layers.37.mlp.experts.121.down_proj", "model.layers.37.mlp.experts.122.down_proj", "model.layers.37.mlp.experts.123.down_proj", "model.layers.37.mlp.experts.124.down_proj", "model.layers.37.mlp.experts.125.down_proj", "model.layers.37.mlp.experts.126.down_proj", "model.layers.37.mlp.experts.127.down_proj", "model.layers.37.mlp.experts.128.down_proj", "model.layers.37.mlp.experts.129.down_proj", "model.layers.37.mlp.experts.130.down_proj", "model.layers.37.mlp.experts.131.down_proj", "model.layers.37.mlp.experts.132.down_proj", "model.layers.37.mlp.experts.133.down_proj", "model.layers.37.mlp.experts.134.down_proj", "model.layers.37.mlp.experts.135.down_proj", "model.layers.37.mlp.experts.136.down_proj", "model.layers.37.mlp.experts.137.down_proj", "model.layers.37.mlp.experts.138.down_proj", "model.layers.37.mlp.experts.139.down_proj", "model.layers.37.mlp.experts.140.down_proj", "model.layers.37.mlp.experts.141.down_proj", "model.layers.37.mlp.experts.142.down_proj", "model.layers.37.mlp.experts.143.down_proj", "model.layers.37.mlp.experts.144.down_proj", "model.layers.37.mlp.experts.145.down_proj", "model.layers.37.mlp.experts.146.down_proj", "model.layers.37.mlp.experts.147.down_proj", "model.layers.37.mlp.experts.148.down_proj", "model.layers.37.mlp.experts.149.down_proj", "model.layers.37.mlp.experts.150.down_proj", "model.layers.37.mlp.experts.151.down_proj", "model.layers.37.mlp.experts.152.down_proj", "model.layers.37.mlp.experts.153.down_proj", "model.layers.37.mlp.experts.154.down_proj", "model.layers.37.mlp.experts.155.down_proj", "model.layers.37.mlp.experts.156.down_proj", "model.layers.37.mlp.experts.157.down_proj", "model.layers.37.mlp.experts.158.down_proj", "model.layers.37.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.2588967112824754e-06, "dbits": 2516582400 } ] }, { "idx": 225, "layers": [ "model.layers.38.self_attn.q_proj" ], "candidates": [ { "dkld": -1.9164435798302293e-06, "dbits": 125829120 } ] }, { "idx": 226, "layers": [ "model.layers.38.self_attn.k_proj", "model.layers.38.self_attn.v_proj" ], "candidates": [ { "dkld": 6.74536568112628e-07, "dbits": 20971520 } ] }, { "idx": 227, "layers": [ "model.layers.38.self_attn.o_proj" ], "candidates": [ { "dkld": -2.9326038202270865e-06, "dbits": 125829120 } ] }, { "idx": 228, "layers": [ "model.layers.38.mlp.shared_experts.gate_proj", "model.layers.38.mlp.shared_experts.up_proj", "model.layers.38.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.2997243902645962e-05, "dbits": 47185920 } ] }, { "idx": 229, "layers": [ "model.layers.38.mlp.experts.0.gate_proj", "model.layers.38.mlp.experts.1.gate_proj", "model.layers.38.mlp.experts.2.gate_proj", "model.layers.38.mlp.experts.3.gate_proj", "model.layers.38.mlp.experts.4.gate_proj", "model.layers.38.mlp.experts.5.gate_proj", "model.layers.38.mlp.experts.6.gate_proj", "model.layers.38.mlp.experts.7.gate_proj", "model.layers.38.mlp.experts.8.gate_proj", "model.layers.38.mlp.experts.9.gate_proj", "model.layers.38.mlp.experts.10.gate_proj", "model.layers.38.mlp.experts.11.gate_proj", "model.layers.38.mlp.experts.12.gate_proj", "model.layers.38.mlp.experts.13.gate_proj", "model.layers.38.mlp.experts.14.gate_proj", "model.layers.38.mlp.experts.15.gate_proj", "model.layers.38.mlp.experts.16.gate_proj", "model.layers.38.mlp.experts.17.gate_proj", "model.layers.38.mlp.experts.18.gate_proj", "model.layers.38.mlp.experts.19.gate_proj", "model.layers.38.mlp.experts.20.gate_proj", "model.layers.38.mlp.experts.21.gate_proj", "model.layers.38.mlp.experts.22.gate_proj", "model.layers.38.mlp.experts.23.gate_proj", "model.layers.38.mlp.experts.24.gate_proj", "model.layers.38.mlp.experts.25.gate_proj", "model.layers.38.mlp.experts.26.gate_proj", "model.layers.38.mlp.experts.27.gate_proj", "model.layers.38.mlp.experts.28.gate_proj", "model.layers.38.mlp.experts.29.gate_proj", "model.layers.38.mlp.experts.30.gate_proj", "model.layers.38.mlp.experts.31.gate_proj", "model.layers.38.mlp.experts.32.gate_proj", "model.layers.38.mlp.experts.33.gate_proj", "model.layers.38.mlp.experts.34.gate_proj", "model.layers.38.mlp.experts.35.gate_proj", "model.layers.38.mlp.experts.36.gate_proj", "model.layers.38.mlp.experts.37.gate_proj", "model.layers.38.mlp.experts.38.gate_proj", "model.layers.38.mlp.experts.39.gate_proj", "model.layers.38.mlp.experts.40.gate_proj", "model.layers.38.mlp.experts.41.gate_proj", "model.layers.38.mlp.experts.42.gate_proj", "model.layers.38.mlp.experts.43.gate_proj", "model.layers.38.mlp.experts.44.gate_proj", "model.layers.38.mlp.experts.45.gate_proj", "model.layers.38.mlp.experts.46.gate_proj", "model.layers.38.mlp.experts.47.gate_proj", "model.layers.38.mlp.experts.48.gate_proj", "model.layers.38.mlp.experts.49.gate_proj", "model.layers.38.mlp.experts.50.gate_proj", "model.layers.38.mlp.experts.51.gate_proj", "model.layers.38.mlp.experts.52.gate_proj", "model.layers.38.mlp.experts.53.gate_proj", "model.layers.38.mlp.experts.54.gate_proj", "model.layers.38.mlp.experts.55.gate_proj", "model.layers.38.mlp.experts.56.gate_proj", "model.layers.38.mlp.experts.57.gate_proj", "model.layers.38.mlp.experts.58.gate_proj", "model.layers.38.mlp.experts.59.gate_proj", "model.layers.38.mlp.experts.60.gate_proj", "model.layers.38.mlp.experts.61.gate_proj", "model.layers.38.mlp.experts.62.gate_proj", "model.layers.38.mlp.experts.63.gate_proj", "model.layers.38.mlp.experts.64.gate_proj", "model.layers.38.mlp.experts.65.gate_proj", "model.layers.38.mlp.experts.66.gate_proj", "model.layers.38.mlp.experts.67.gate_proj", "model.layers.38.mlp.experts.68.gate_proj", "model.layers.38.mlp.experts.69.gate_proj", "model.layers.38.mlp.experts.70.gate_proj", "model.layers.38.mlp.experts.71.gate_proj", "model.layers.38.mlp.experts.72.gate_proj", "model.layers.38.mlp.experts.73.gate_proj", "model.layers.38.mlp.experts.74.gate_proj", "model.layers.38.mlp.experts.75.gate_proj", "model.layers.38.mlp.experts.76.gate_proj", "model.layers.38.mlp.experts.77.gate_proj", "model.layers.38.mlp.experts.78.gate_proj", "model.layers.38.mlp.experts.79.gate_proj", "model.layers.38.mlp.experts.80.gate_proj", "model.layers.38.mlp.experts.81.gate_proj", "model.layers.38.mlp.experts.82.gate_proj", "model.layers.38.mlp.experts.83.gate_proj", "model.layers.38.mlp.experts.84.gate_proj", "model.layers.38.mlp.experts.85.gate_proj", "model.layers.38.mlp.experts.86.gate_proj", "model.layers.38.mlp.experts.87.gate_proj", "model.layers.38.mlp.experts.88.gate_proj", "model.layers.38.mlp.experts.89.gate_proj", "model.layers.38.mlp.experts.90.gate_proj", "model.layers.38.mlp.experts.91.gate_proj", "model.layers.38.mlp.experts.92.gate_proj", "model.layers.38.mlp.experts.93.gate_proj", "model.layers.38.mlp.experts.94.gate_proj", "model.layers.38.mlp.experts.95.gate_proj", "model.layers.38.mlp.experts.96.gate_proj", "model.layers.38.mlp.experts.97.gate_proj", "model.layers.38.mlp.experts.98.gate_proj", "model.layers.38.mlp.experts.99.gate_proj", "model.layers.38.mlp.experts.100.gate_proj", "model.layers.38.mlp.experts.101.gate_proj", "model.layers.38.mlp.experts.102.gate_proj", "model.layers.38.mlp.experts.103.gate_proj", "model.layers.38.mlp.experts.104.gate_proj", "model.layers.38.mlp.experts.105.gate_proj", "model.layers.38.mlp.experts.106.gate_proj", "model.layers.38.mlp.experts.107.gate_proj", "model.layers.38.mlp.experts.108.gate_proj", "model.layers.38.mlp.experts.109.gate_proj", "model.layers.38.mlp.experts.110.gate_proj", "model.layers.38.mlp.experts.111.gate_proj", "model.layers.38.mlp.experts.112.gate_proj", "model.layers.38.mlp.experts.113.gate_proj", "model.layers.38.mlp.experts.114.gate_proj", "model.layers.38.mlp.experts.115.gate_proj", "model.layers.38.mlp.experts.116.gate_proj", "model.layers.38.mlp.experts.117.gate_proj", "model.layers.38.mlp.experts.118.gate_proj", "model.layers.38.mlp.experts.119.gate_proj", "model.layers.38.mlp.experts.120.gate_proj", "model.layers.38.mlp.experts.121.gate_proj", "model.layers.38.mlp.experts.122.gate_proj", "model.layers.38.mlp.experts.123.gate_proj", "model.layers.38.mlp.experts.124.gate_proj", "model.layers.38.mlp.experts.125.gate_proj", "model.layers.38.mlp.experts.126.gate_proj", "model.layers.38.mlp.experts.127.gate_proj", "model.layers.38.mlp.experts.128.gate_proj", "model.layers.38.mlp.experts.129.gate_proj", "model.layers.38.mlp.experts.130.gate_proj", "model.layers.38.mlp.experts.131.gate_proj", "model.layers.38.mlp.experts.132.gate_proj", "model.layers.38.mlp.experts.133.gate_proj", "model.layers.38.mlp.experts.134.gate_proj", "model.layers.38.mlp.experts.135.gate_proj", "model.layers.38.mlp.experts.136.gate_proj", "model.layers.38.mlp.experts.137.gate_proj", "model.layers.38.mlp.experts.138.gate_proj", "model.layers.38.mlp.experts.139.gate_proj", "model.layers.38.mlp.experts.140.gate_proj", "model.layers.38.mlp.experts.141.gate_proj", "model.layers.38.mlp.experts.142.gate_proj", "model.layers.38.mlp.experts.143.gate_proj", "model.layers.38.mlp.experts.144.gate_proj", "model.layers.38.mlp.experts.145.gate_proj", "model.layers.38.mlp.experts.146.gate_proj", "model.layers.38.mlp.experts.147.gate_proj", "model.layers.38.mlp.experts.148.gate_proj", "model.layers.38.mlp.experts.149.gate_proj", "model.layers.38.mlp.experts.150.gate_proj", "model.layers.38.mlp.experts.151.gate_proj", "model.layers.38.mlp.experts.152.gate_proj", "model.layers.38.mlp.experts.153.gate_proj", "model.layers.38.mlp.experts.154.gate_proj", "model.layers.38.mlp.experts.155.gate_proj", "model.layers.38.mlp.experts.156.gate_proj", "model.layers.38.mlp.experts.157.gate_proj", "model.layers.38.mlp.experts.158.gate_proj", "model.layers.38.mlp.experts.159.gate_proj", "model.layers.38.mlp.experts.0.up_proj", "model.layers.38.mlp.experts.1.up_proj", "model.layers.38.mlp.experts.2.up_proj", "model.layers.38.mlp.experts.3.up_proj", "model.layers.38.mlp.experts.4.up_proj", "model.layers.38.mlp.experts.5.up_proj", "model.layers.38.mlp.experts.6.up_proj", "model.layers.38.mlp.experts.7.up_proj", "model.layers.38.mlp.experts.8.up_proj", "model.layers.38.mlp.experts.9.up_proj", "model.layers.38.mlp.experts.10.up_proj", "model.layers.38.mlp.experts.11.up_proj", "model.layers.38.mlp.experts.12.up_proj", "model.layers.38.mlp.experts.13.up_proj", "model.layers.38.mlp.experts.14.up_proj", "model.layers.38.mlp.experts.15.up_proj", "model.layers.38.mlp.experts.16.up_proj", "model.layers.38.mlp.experts.17.up_proj", "model.layers.38.mlp.experts.18.up_proj", "model.layers.38.mlp.experts.19.up_proj", "model.layers.38.mlp.experts.20.up_proj", "model.layers.38.mlp.experts.21.up_proj", "model.layers.38.mlp.experts.22.up_proj", "model.layers.38.mlp.experts.23.up_proj", "model.layers.38.mlp.experts.24.up_proj", "model.layers.38.mlp.experts.25.up_proj", "model.layers.38.mlp.experts.26.up_proj", "model.layers.38.mlp.experts.27.up_proj", "model.layers.38.mlp.experts.28.up_proj", "model.layers.38.mlp.experts.29.up_proj", "model.layers.38.mlp.experts.30.up_proj", "model.layers.38.mlp.experts.31.up_proj", "model.layers.38.mlp.experts.32.up_proj", "model.layers.38.mlp.experts.33.up_proj", "model.layers.38.mlp.experts.34.up_proj", "model.layers.38.mlp.experts.35.up_proj", "model.layers.38.mlp.experts.36.up_proj", "model.layers.38.mlp.experts.37.up_proj", "model.layers.38.mlp.experts.38.up_proj", "model.layers.38.mlp.experts.39.up_proj", "model.layers.38.mlp.experts.40.up_proj", "model.layers.38.mlp.experts.41.up_proj", "model.layers.38.mlp.experts.42.up_proj", "model.layers.38.mlp.experts.43.up_proj", "model.layers.38.mlp.experts.44.up_proj", "model.layers.38.mlp.experts.45.up_proj", "model.layers.38.mlp.experts.46.up_proj", "model.layers.38.mlp.experts.47.up_proj", "model.layers.38.mlp.experts.48.up_proj", "model.layers.38.mlp.experts.49.up_proj", "model.layers.38.mlp.experts.50.up_proj", "model.layers.38.mlp.experts.51.up_proj", "model.layers.38.mlp.experts.52.up_proj", "model.layers.38.mlp.experts.53.up_proj", "model.layers.38.mlp.experts.54.up_proj", "model.layers.38.mlp.experts.55.up_proj", "model.layers.38.mlp.experts.56.up_proj", "model.layers.38.mlp.experts.57.up_proj", "model.layers.38.mlp.experts.58.up_proj", "model.layers.38.mlp.experts.59.up_proj", "model.layers.38.mlp.experts.60.up_proj", "model.layers.38.mlp.experts.61.up_proj", "model.layers.38.mlp.experts.62.up_proj", "model.layers.38.mlp.experts.63.up_proj", "model.layers.38.mlp.experts.64.up_proj", "model.layers.38.mlp.experts.65.up_proj", "model.layers.38.mlp.experts.66.up_proj", "model.layers.38.mlp.experts.67.up_proj", "model.layers.38.mlp.experts.68.up_proj", "model.layers.38.mlp.experts.69.up_proj", "model.layers.38.mlp.experts.70.up_proj", "model.layers.38.mlp.experts.71.up_proj", "model.layers.38.mlp.experts.72.up_proj", "model.layers.38.mlp.experts.73.up_proj", "model.layers.38.mlp.experts.74.up_proj", "model.layers.38.mlp.experts.75.up_proj", "model.layers.38.mlp.experts.76.up_proj", "model.layers.38.mlp.experts.77.up_proj", "model.layers.38.mlp.experts.78.up_proj", "model.layers.38.mlp.experts.79.up_proj", "model.layers.38.mlp.experts.80.up_proj", "model.layers.38.mlp.experts.81.up_proj", "model.layers.38.mlp.experts.82.up_proj", "model.layers.38.mlp.experts.83.up_proj", "model.layers.38.mlp.experts.84.up_proj", "model.layers.38.mlp.experts.85.up_proj", "model.layers.38.mlp.experts.86.up_proj", "model.layers.38.mlp.experts.87.up_proj", "model.layers.38.mlp.experts.88.up_proj", "model.layers.38.mlp.experts.89.up_proj", "model.layers.38.mlp.experts.90.up_proj", "model.layers.38.mlp.experts.91.up_proj", "model.layers.38.mlp.experts.92.up_proj", "model.layers.38.mlp.experts.93.up_proj", "model.layers.38.mlp.experts.94.up_proj", "model.layers.38.mlp.experts.95.up_proj", "model.layers.38.mlp.experts.96.up_proj", "model.layers.38.mlp.experts.97.up_proj", "model.layers.38.mlp.experts.98.up_proj", "model.layers.38.mlp.experts.99.up_proj", "model.layers.38.mlp.experts.100.up_proj", "model.layers.38.mlp.experts.101.up_proj", "model.layers.38.mlp.experts.102.up_proj", "model.layers.38.mlp.experts.103.up_proj", "model.layers.38.mlp.experts.104.up_proj", "model.layers.38.mlp.experts.105.up_proj", "model.layers.38.mlp.experts.106.up_proj", "model.layers.38.mlp.experts.107.up_proj", "model.layers.38.mlp.experts.108.up_proj", "model.layers.38.mlp.experts.109.up_proj", "model.layers.38.mlp.experts.110.up_proj", "model.layers.38.mlp.experts.111.up_proj", "model.layers.38.mlp.experts.112.up_proj", "model.layers.38.mlp.experts.113.up_proj", "model.layers.38.mlp.experts.114.up_proj", "model.layers.38.mlp.experts.115.up_proj", "model.layers.38.mlp.experts.116.up_proj", "model.layers.38.mlp.experts.117.up_proj", "model.layers.38.mlp.experts.118.up_proj", "model.layers.38.mlp.experts.119.up_proj", "model.layers.38.mlp.experts.120.up_proj", "model.layers.38.mlp.experts.121.up_proj", "model.layers.38.mlp.experts.122.up_proj", "model.layers.38.mlp.experts.123.up_proj", "model.layers.38.mlp.experts.124.up_proj", "model.layers.38.mlp.experts.125.up_proj", "model.layers.38.mlp.experts.126.up_proj", "model.layers.38.mlp.experts.127.up_proj", "model.layers.38.mlp.experts.128.up_proj", "model.layers.38.mlp.experts.129.up_proj", "model.layers.38.mlp.experts.130.up_proj", "model.layers.38.mlp.experts.131.up_proj", "model.layers.38.mlp.experts.132.up_proj", "model.layers.38.mlp.experts.133.up_proj", "model.layers.38.mlp.experts.134.up_proj", "model.layers.38.mlp.experts.135.up_proj", "model.layers.38.mlp.experts.136.up_proj", "model.layers.38.mlp.experts.137.up_proj", "model.layers.38.mlp.experts.138.up_proj", "model.layers.38.mlp.experts.139.up_proj", "model.layers.38.mlp.experts.140.up_proj", "model.layers.38.mlp.experts.141.up_proj", "model.layers.38.mlp.experts.142.up_proj", "model.layers.38.mlp.experts.143.up_proj", "model.layers.38.mlp.experts.144.up_proj", "model.layers.38.mlp.experts.145.up_proj", "model.layers.38.mlp.experts.146.up_proj", "model.layers.38.mlp.experts.147.up_proj", "model.layers.38.mlp.experts.148.up_proj", "model.layers.38.mlp.experts.149.up_proj", "model.layers.38.mlp.experts.150.up_proj", "model.layers.38.mlp.experts.151.up_proj", "model.layers.38.mlp.experts.152.up_proj", "model.layers.38.mlp.experts.153.up_proj", "model.layers.38.mlp.experts.154.up_proj", "model.layers.38.mlp.experts.155.up_proj", "model.layers.38.mlp.experts.156.up_proj", "model.layers.38.mlp.experts.157.up_proj", "model.layers.38.mlp.experts.158.up_proj", "model.layers.38.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.927815800532785e-06, "dbits": 5033164800 } ] }, { "idx": 230, "layers": [ "model.layers.38.mlp.experts.0.down_proj", "model.layers.38.mlp.experts.1.down_proj", "model.layers.38.mlp.experts.2.down_proj", "model.layers.38.mlp.experts.3.down_proj", "model.layers.38.mlp.experts.4.down_proj", "model.layers.38.mlp.experts.5.down_proj", "model.layers.38.mlp.experts.6.down_proj", "model.layers.38.mlp.experts.7.down_proj", "model.layers.38.mlp.experts.8.down_proj", "model.layers.38.mlp.experts.9.down_proj", "model.layers.38.mlp.experts.10.down_proj", "model.layers.38.mlp.experts.11.down_proj", "model.layers.38.mlp.experts.12.down_proj", "model.layers.38.mlp.experts.13.down_proj", "model.layers.38.mlp.experts.14.down_proj", "model.layers.38.mlp.experts.15.down_proj", "model.layers.38.mlp.experts.16.down_proj", "model.layers.38.mlp.experts.17.down_proj", "model.layers.38.mlp.experts.18.down_proj", "model.layers.38.mlp.experts.19.down_proj", "model.layers.38.mlp.experts.20.down_proj", "model.layers.38.mlp.experts.21.down_proj", "model.layers.38.mlp.experts.22.down_proj", "model.layers.38.mlp.experts.23.down_proj", "model.layers.38.mlp.experts.24.down_proj", "model.layers.38.mlp.experts.25.down_proj", "model.layers.38.mlp.experts.26.down_proj", "model.layers.38.mlp.experts.27.down_proj", "model.layers.38.mlp.experts.28.down_proj", "model.layers.38.mlp.experts.29.down_proj", "model.layers.38.mlp.experts.30.down_proj", "model.layers.38.mlp.experts.31.down_proj", "model.layers.38.mlp.experts.32.down_proj", "model.layers.38.mlp.experts.33.down_proj", "model.layers.38.mlp.experts.34.down_proj", "model.layers.38.mlp.experts.35.down_proj", "model.layers.38.mlp.experts.36.down_proj", "model.layers.38.mlp.experts.37.down_proj", "model.layers.38.mlp.experts.38.down_proj", "model.layers.38.mlp.experts.39.down_proj", "model.layers.38.mlp.experts.40.down_proj", "model.layers.38.mlp.experts.41.down_proj", "model.layers.38.mlp.experts.42.down_proj", "model.layers.38.mlp.experts.43.down_proj", "model.layers.38.mlp.experts.44.down_proj", "model.layers.38.mlp.experts.45.down_proj", "model.layers.38.mlp.experts.46.down_proj", "model.layers.38.mlp.experts.47.down_proj", "model.layers.38.mlp.experts.48.down_proj", "model.layers.38.mlp.experts.49.down_proj", "model.layers.38.mlp.experts.50.down_proj", "model.layers.38.mlp.experts.51.down_proj", "model.layers.38.mlp.experts.52.down_proj", "model.layers.38.mlp.experts.53.down_proj", "model.layers.38.mlp.experts.54.down_proj", "model.layers.38.mlp.experts.55.down_proj", "model.layers.38.mlp.experts.56.down_proj", "model.layers.38.mlp.experts.57.down_proj", "model.layers.38.mlp.experts.58.down_proj", "model.layers.38.mlp.experts.59.down_proj", "model.layers.38.mlp.experts.60.down_proj", "model.layers.38.mlp.experts.61.down_proj", "model.layers.38.mlp.experts.62.down_proj", "model.layers.38.mlp.experts.63.down_proj", "model.layers.38.mlp.experts.64.down_proj", "model.layers.38.mlp.experts.65.down_proj", "model.layers.38.mlp.experts.66.down_proj", "model.layers.38.mlp.experts.67.down_proj", "model.layers.38.mlp.experts.68.down_proj", "model.layers.38.mlp.experts.69.down_proj", "model.layers.38.mlp.experts.70.down_proj", "model.layers.38.mlp.experts.71.down_proj", "model.layers.38.mlp.experts.72.down_proj", "model.layers.38.mlp.experts.73.down_proj", "model.layers.38.mlp.experts.74.down_proj", "model.layers.38.mlp.experts.75.down_proj", "model.layers.38.mlp.experts.76.down_proj", "model.layers.38.mlp.experts.77.down_proj", "model.layers.38.mlp.experts.78.down_proj", "model.layers.38.mlp.experts.79.down_proj", "model.layers.38.mlp.experts.80.down_proj", "model.layers.38.mlp.experts.81.down_proj", "model.layers.38.mlp.experts.82.down_proj", "model.layers.38.mlp.experts.83.down_proj", "model.layers.38.mlp.experts.84.down_proj", "model.layers.38.mlp.experts.85.down_proj", "model.layers.38.mlp.experts.86.down_proj", "model.layers.38.mlp.experts.87.down_proj", "model.layers.38.mlp.experts.88.down_proj", "model.layers.38.mlp.experts.89.down_proj", "model.layers.38.mlp.experts.90.down_proj", "model.layers.38.mlp.experts.91.down_proj", "model.layers.38.mlp.experts.92.down_proj", "model.layers.38.mlp.experts.93.down_proj", "model.layers.38.mlp.experts.94.down_proj", "model.layers.38.mlp.experts.95.down_proj", "model.layers.38.mlp.experts.96.down_proj", "model.layers.38.mlp.experts.97.down_proj", "model.layers.38.mlp.experts.98.down_proj", "model.layers.38.mlp.experts.99.down_proj", "model.layers.38.mlp.experts.100.down_proj", "model.layers.38.mlp.experts.101.down_proj", "model.layers.38.mlp.experts.102.down_proj", "model.layers.38.mlp.experts.103.down_proj", "model.layers.38.mlp.experts.104.down_proj", "model.layers.38.mlp.experts.105.down_proj", "model.layers.38.mlp.experts.106.down_proj", "model.layers.38.mlp.experts.107.down_proj", "model.layers.38.mlp.experts.108.down_proj", "model.layers.38.mlp.experts.109.down_proj", "model.layers.38.mlp.experts.110.down_proj", "model.layers.38.mlp.experts.111.down_proj", "model.layers.38.mlp.experts.112.down_proj", "model.layers.38.mlp.experts.113.down_proj", "model.layers.38.mlp.experts.114.down_proj", "model.layers.38.mlp.experts.115.down_proj", "model.layers.38.mlp.experts.116.down_proj", "model.layers.38.mlp.experts.117.down_proj", "model.layers.38.mlp.experts.118.down_proj", "model.layers.38.mlp.experts.119.down_proj", "model.layers.38.mlp.experts.120.down_proj", "model.layers.38.mlp.experts.121.down_proj", "model.layers.38.mlp.experts.122.down_proj", "model.layers.38.mlp.experts.123.down_proj", "model.layers.38.mlp.experts.124.down_proj", "model.layers.38.mlp.experts.125.down_proj", "model.layers.38.mlp.experts.126.down_proj", "model.layers.38.mlp.experts.127.down_proj", "model.layers.38.mlp.experts.128.down_proj", "model.layers.38.mlp.experts.129.down_proj", "model.layers.38.mlp.experts.130.down_proj", "model.layers.38.mlp.experts.131.down_proj", "model.layers.38.mlp.experts.132.down_proj", "model.layers.38.mlp.experts.133.down_proj", "model.layers.38.mlp.experts.134.down_proj", "model.layers.38.mlp.experts.135.down_proj", "model.layers.38.mlp.experts.136.down_proj", "model.layers.38.mlp.experts.137.down_proj", "model.layers.38.mlp.experts.138.down_proj", "model.layers.38.mlp.experts.139.down_proj", "model.layers.38.mlp.experts.140.down_proj", "model.layers.38.mlp.experts.141.down_proj", "model.layers.38.mlp.experts.142.down_proj", "model.layers.38.mlp.experts.143.down_proj", "model.layers.38.mlp.experts.144.down_proj", "model.layers.38.mlp.experts.145.down_proj", "model.layers.38.mlp.experts.146.down_proj", "model.layers.38.mlp.experts.147.down_proj", "model.layers.38.mlp.experts.148.down_proj", "model.layers.38.mlp.experts.149.down_proj", "model.layers.38.mlp.experts.150.down_proj", "model.layers.38.mlp.experts.151.down_proj", "model.layers.38.mlp.experts.152.down_proj", "model.layers.38.mlp.experts.153.down_proj", "model.layers.38.mlp.experts.154.down_proj", "model.layers.38.mlp.experts.155.down_proj", "model.layers.38.mlp.experts.156.down_proj", "model.layers.38.mlp.experts.157.down_proj", "model.layers.38.mlp.experts.158.down_proj", "model.layers.38.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.1415337212383964e-06, "dbits": 2516582400 } ] }, { "idx": 231, "layers": [ "model.layers.39.self_attn.q_proj" ], "candidates": [ { "dkld": -9.865761967375875e-07, "dbits": 125829120 } ] }, { "idx": 232, "layers": [ "model.layers.39.self_attn.k_proj", "model.layers.39.self_attn.v_proj" ], "candidates": [ { "dkld": 5.61903580091898e-06, "dbits": 20971520 } ] }, { "idx": 233, "layers": [ "model.layers.39.self_attn.o_proj" ], "candidates": [ { "dkld": -3.6594487028196237e-06, "dbits": 125829120 } ] }, { "idx": 234, "layers": [ "model.layers.39.mlp.shared_experts.gate_proj", "model.layers.39.mlp.shared_experts.up_proj", "model.layers.39.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.2846518075093844e-06, "dbits": 47185920 } ] }, { "idx": 235, "layers": [ "model.layers.39.mlp.experts.0.gate_proj", "model.layers.39.mlp.experts.1.gate_proj", "model.layers.39.mlp.experts.2.gate_proj", "model.layers.39.mlp.experts.3.gate_proj", "model.layers.39.mlp.experts.4.gate_proj", "model.layers.39.mlp.experts.5.gate_proj", "model.layers.39.mlp.experts.6.gate_proj", "model.layers.39.mlp.experts.7.gate_proj", "model.layers.39.mlp.experts.8.gate_proj", "model.layers.39.mlp.experts.9.gate_proj", "model.layers.39.mlp.experts.10.gate_proj", "model.layers.39.mlp.experts.11.gate_proj", "model.layers.39.mlp.experts.12.gate_proj", "model.layers.39.mlp.experts.13.gate_proj", "model.layers.39.mlp.experts.14.gate_proj", "model.layers.39.mlp.experts.15.gate_proj", "model.layers.39.mlp.experts.16.gate_proj", "model.layers.39.mlp.experts.17.gate_proj", "model.layers.39.mlp.experts.18.gate_proj", "model.layers.39.mlp.experts.19.gate_proj", "model.layers.39.mlp.experts.20.gate_proj", "model.layers.39.mlp.experts.21.gate_proj", "model.layers.39.mlp.experts.22.gate_proj", "model.layers.39.mlp.experts.23.gate_proj", "model.layers.39.mlp.experts.24.gate_proj", "model.layers.39.mlp.experts.25.gate_proj", "model.layers.39.mlp.experts.26.gate_proj", "model.layers.39.mlp.experts.27.gate_proj", "model.layers.39.mlp.experts.28.gate_proj", "model.layers.39.mlp.experts.29.gate_proj", "model.layers.39.mlp.experts.30.gate_proj", "model.layers.39.mlp.experts.31.gate_proj", "model.layers.39.mlp.experts.32.gate_proj", "model.layers.39.mlp.experts.33.gate_proj", "model.layers.39.mlp.experts.34.gate_proj", "model.layers.39.mlp.experts.35.gate_proj", "model.layers.39.mlp.experts.36.gate_proj", "model.layers.39.mlp.experts.37.gate_proj", "model.layers.39.mlp.experts.38.gate_proj", "model.layers.39.mlp.experts.39.gate_proj", "model.layers.39.mlp.experts.40.gate_proj", "model.layers.39.mlp.experts.41.gate_proj", "model.layers.39.mlp.experts.42.gate_proj", "model.layers.39.mlp.experts.43.gate_proj", "model.layers.39.mlp.experts.44.gate_proj", "model.layers.39.mlp.experts.45.gate_proj", "model.layers.39.mlp.experts.46.gate_proj", "model.layers.39.mlp.experts.47.gate_proj", "model.layers.39.mlp.experts.48.gate_proj", "model.layers.39.mlp.experts.49.gate_proj", "model.layers.39.mlp.experts.50.gate_proj", "model.layers.39.mlp.experts.51.gate_proj", "model.layers.39.mlp.experts.52.gate_proj", "model.layers.39.mlp.experts.53.gate_proj", "model.layers.39.mlp.experts.54.gate_proj", "model.layers.39.mlp.experts.55.gate_proj", "model.layers.39.mlp.experts.56.gate_proj", "model.layers.39.mlp.experts.57.gate_proj", "model.layers.39.mlp.experts.58.gate_proj", "model.layers.39.mlp.experts.59.gate_proj", "model.layers.39.mlp.experts.60.gate_proj", "model.layers.39.mlp.experts.61.gate_proj", "model.layers.39.mlp.experts.62.gate_proj", "model.layers.39.mlp.experts.63.gate_proj", "model.layers.39.mlp.experts.64.gate_proj", "model.layers.39.mlp.experts.65.gate_proj", "model.layers.39.mlp.experts.66.gate_proj", "model.layers.39.mlp.experts.67.gate_proj", "model.layers.39.mlp.experts.68.gate_proj", "model.layers.39.mlp.experts.69.gate_proj", "model.layers.39.mlp.experts.70.gate_proj", "model.layers.39.mlp.experts.71.gate_proj", "model.layers.39.mlp.experts.72.gate_proj", "model.layers.39.mlp.experts.73.gate_proj", "model.layers.39.mlp.experts.74.gate_proj", "model.layers.39.mlp.experts.75.gate_proj", "model.layers.39.mlp.experts.76.gate_proj", "model.layers.39.mlp.experts.77.gate_proj", "model.layers.39.mlp.experts.78.gate_proj", "model.layers.39.mlp.experts.79.gate_proj", "model.layers.39.mlp.experts.80.gate_proj", "model.layers.39.mlp.experts.81.gate_proj", "model.layers.39.mlp.experts.82.gate_proj", "model.layers.39.mlp.experts.83.gate_proj", "model.layers.39.mlp.experts.84.gate_proj", "model.layers.39.mlp.experts.85.gate_proj", "model.layers.39.mlp.experts.86.gate_proj", "model.layers.39.mlp.experts.87.gate_proj", "model.layers.39.mlp.experts.88.gate_proj", "model.layers.39.mlp.experts.89.gate_proj", "model.layers.39.mlp.experts.90.gate_proj", "model.layers.39.mlp.experts.91.gate_proj", "model.layers.39.mlp.experts.92.gate_proj", "model.layers.39.mlp.experts.93.gate_proj", "model.layers.39.mlp.experts.94.gate_proj", "model.layers.39.mlp.experts.95.gate_proj", "model.layers.39.mlp.experts.96.gate_proj", "model.layers.39.mlp.experts.97.gate_proj", "model.layers.39.mlp.experts.98.gate_proj", "model.layers.39.mlp.experts.99.gate_proj", "model.layers.39.mlp.experts.100.gate_proj", "model.layers.39.mlp.experts.101.gate_proj", "model.layers.39.mlp.experts.102.gate_proj", "model.layers.39.mlp.experts.103.gate_proj", "model.layers.39.mlp.experts.104.gate_proj", "model.layers.39.mlp.experts.105.gate_proj", "model.layers.39.mlp.experts.106.gate_proj", "model.layers.39.mlp.experts.107.gate_proj", "model.layers.39.mlp.experts.108.gate_proj", "model.layers.39.mlp.experts.109.gate_proj", "model.layers.39.mlp.experts.110.gate_proj", "model.layers.39.mlp.experts.111.gate_proj", "model.layers.39.mlp.experts.112.gate_proj", "model.layers.39.mlp.experts.113.gate_proj", "model.layers.39.mlp.experts.114.gate_proj", "model.layers.39.mlp.experts.115.gate_proj", "model.layers.39.mlp.experts.116.gate_proj", "model.layers.39.mlp.experts.117.gate_proj", "model.layers.39.mlp.experts.118.gate_proj", "model.layers.39.mlp.experts.119.gate_proj", "model.layers.39.mlp.experts.120.gate_proj", "model.layers.39.mlp.experts.121.gate_proj", "model.layers.39.mlp.experts.122.gate_proj", "model.layers.39.mlp.experts.123.gate_proj", "model.layers.39.mlp.experts.124.gate_proj", "model.layers.39.mlp.experts.125.gate_proj", "model.layers.39.mlp.experts.126.gate_proj", "model.layers.39.mlp.experts.127.gate_proj", "model.layers.39.mlp.experts.128.gate_proj", "model.layers.39.mlp.experts.129.gate_proj", "model.layers.39.mlp.experts.130.gate_proj", "model.layers.39.mlp.experts.131.gate_proj", "model.layers.39.mlp.experts.132.gate_proj", "model.layers.39.mlp.experts.133.gate_proj", "model.layers.39.mlp.experts.134.gate_proj", "model.layers.39.mlp.experts.135.gate_proj", "model.layers.39.mlp.experts.136.gate_proj", "model.layers.39.mlp.experts.137.gate_proj", "model.layers.39.mlp.experts.138.gate_proj", "model.layers.39.mlp.experts.139.gate_proj", "model.layers.39.mlp.experts.140.gate_proj", "model.layers.39.mlp.experts.141.gate_proj", "model.layers.39.mlp.experts.142.gate_proj", "model.layers.39.mlp.experts.143.gate_proj", "model.layers.39.mlp.experts.144.gate_proj", "model.layers.39.mlp.experts.145.gate_proj", "model.layers.39.mlp.experts.146.gate_proj", "model.layers.39.mlp.experts.147.gate_proj", "model.layers.39.mlp.experts.148.gate_proj", "model.layers.39.mlp.experts.149.gate_proj", "model.layers.39.mlp.experts.150.gate_proj", "model.layers.39.mlp.experts.151.gate_proj", "model.layers.39.mlp.experts.152.gate_proj", "model.layers.39.mlp.experts.153.gate_proj", "model.layers.39.mlp.experts.154.gate_proj", "model.layers.39.mlp.experts.155.gate_proj", "model.layers.39.mlp.experts.156.gate_proj", "model.layers.39.mlp.experts.157.gate_proj", "model.layers.39.mlp.experts.158.gate_proj", "model.layers.39.mlp.experts.159.gate_proj", "model.layers.39.mlp.experts.0.up_proj", "model.layers.39.mlp.experts.1.up_proj", "model.layers.39.mlp.experts.2.up_proj", "model.layers.39.mlp.experts.3.up_proj", "model.layers.39.mlp.experts.4.up_proj", "model.layers.39.mlp.experts.5.up_proj", "model.layers.39.mlp.experts.6.up_proj", "model.layers.39.mlp.experts.7.up_proj", "model.layers.39.mlp.experts.8.up_proj", "model.layers.39.mlp.experts.9.up_proj", "model.layers.39.mlp.experts.10.up_proj", "model.layers.39.mlp.experts.11.up_proj", "model.layers.39.mlp.experts.12.up_proj", "model.layers.39.mlp.experts.13.up_proj", "model.layers.39.mlp.experts.14.up_proj", "model.layers.39.mlp.experts.15.up_proj", "model.layers.39.mlp.experts.16.up_proj", "model.layers.39.mlp.experts.17.up_proj", "model.layers.39.mlp.experts.18.up_proj", "model.layers.39.mlp.experts.19.up_proj", "model.layers.39.mlp.experts.20.up_proj", "model.layers.39.mlp.experts.21.up_proj", "model.layers.39.mlp.experts.22.up_proj", "model.layers.39.mlp.experts.23.up_proj", "model.layers.39.mlp.experts.24.up_proj", "model.layers.39.mlp.experts.25.up_proj", "model.layers.39.mlp.experts.26.up_proj", "model.layers.39.mlp.experts.27.up_proj", "model.layers.39.mlp.experts.28.up_proj", "model.layers.39.mlp.experts.29.up_proj", "model.layers.39.mlp.experts.30.up_proj", "model.layers.39.mlp.experts.31.up_proj", "model.layers.39.mlp.experts.32.up_proj", "model.layers.39.mlp.experts.33.up_proj", "model.layers.39.mlp.experts.34.up_proj", "model.layers.39.mlp.experts.35.up_proj", "model.layers.39.mlp.experts.36.up_proj", "model.layers.39.mlp.experts.37.up_proj", "model.layers.39.mlp.experts.38.up_proj", "model.layers.39.mlp.experts.39.up_proj", "model.layers.39.mlp.experts.40.up_proj", "model.layers.39.mlp.experts.41.up_proj", "model.layers.39.mlp.experts.42.up_proj", "model.layers.39.mlp.experts.43.up_proj", "model.layers.39.mlp.experts.44.up_proj", "model.layers.39.mlp.experts.45.up_proj", "model.layers.39.mlp.experts.46.up_proj", "model.layers.39.mlp.experts.47.up_proj", "model.layers.39.mlp.experts.48.up_proj", "model.layers.39.mlp.experts.49.up_proj", "model.layers.39.mlp.experts.50.up_proj", "model.layers.39.mlp.experts.51.up_proj", "model.layers.39.mlp.experts.52.up_proj", "model.layers.39.mlp.experts.53.up_proj", "model.layers.39.mlp.experts.54.up_proj", "model.layers.39.mlp.experts.55.up_proj", "model.layers.39.mlp.experts.56.up_proj", "model.layers.39.mlp.experts.57.up_proj", "model.layers.39.mlp.experts.58.up_proj", "model.layers.39.mlp.experts.59.up_proj", "model.layers.39.mlp.experts.60.up_proj", "model.layers.39.mlp.experts.61.up_proj", "model.layers.39.mlp.experts.62.up_proj", "model.layers.39.mlp.experts.63.up_proj", "model.layers.39.mlp.experts.64.up_proj", "model.layers.39.mlp.experts.65.up_proj", "model.layers.39.mlp.experts.66.up_proj", "model.layers.39.mlp.experts.67.up_proj", "model.layers.39.mlp.experts.68.up_proj", "model.layers.39.mlp.experts.69.up_proj", "model.layers.39.mlp.experts.70.up_proj", "model.layers.39.mlp.experts.71.up_proj", "model.layers.39.mlp.experts.72.up_proj", "model.layers.39.mlp.experts.73.up_proj", "model.layers.39.mlp.experts.74.up_proj", "model.layers.39.mlp.experts.75.up_proj", "model.layers.39.mlp.experts.76.up_proj", "model.layers.39.mlp.experts.77.up_proj", "model.layers.39.mlp.experts.78.up_proj", "model.layers.39.mlp.experts.79.up_proj", "model.layers.39.mlp.experts.80.up_proj", "model.layers.39.mlp.experts.81.up_proj", "model.layers.39.mlp.experts.82.up_proj", "model.layers.39.mlp.experts.83.up_proj", "model.layers.39.mlp.experts.84.up_proj", "model.layers.39.mlp.experts.85.up_proj", "model.layers.39.mlp.experts.86.up_proj", "model.layers.39.mlp.experts.87.up_proj", "model.layers.39.mlp.experts.88.up_proj", "model.layers.39.mlp.experts.89.up_proj", "model.layers.39.mlp.experts.90.up_proj", "model.layers.39.mlp.experts.91.up_proj", "model.layers.39.mlp.experts.92.up_proj", "model.layers.39.mlp.experts.93.up_proj", "model.layers.39.mlp.experts.94.up_proj", "model.layers.39.mlp.experts.95.up_proj", "model.layers.39.mlp.experts.96.up_proj", "model.layers.39.mlp.experts.97.up_proj", "model.layers.39.mlp.experts.98.up_proj", "model.layers.39.mlp.experts.99.up_proj", "model.layers.39.mlp.experts.100.up_proj", "model.layers.39.mlp.experts.101.up_proj", "model.layers.39.mlp.experts.102.up_proj", "model.layers.39.mlp.experts.103.up_proj", "model.layers.39.mlp.experts.104.up_proj", "model.layers.39.mlp.experts.105.up_proj", "model.layers.39.mlp.experts.106.up_proj", "model.layers.39.mlp.experts.107.up_proj", "model.layers.39.mlp.experts.108.up_proj", "model.layers.39.mlp.experts.109.up_proj", "model.layers.39.mlp.experts.110.up_proj", "model.layers.39.mlp.experts.111.up_proj", "model.layers.39.mlp.experts.112.up_proj", "model.layers.39.mlp.experts.113.up_proj", "model.layers.39.mlp.experts.114.up_proj", "model.layers.39.mlp.experts.115.up_proj", "model.layers.39.mlp.experts.116.up_proj", "model.layers.39.mlp.experts.117.up_proj", "model.layers.39.mlp.experts.118.up_proj", "model.layers.39.mlp.experts.119.up_proj", "model.layers.39.mlp.experts.120.up_proj", "model.layers.39.mlp.experts.121.up_proj", "model.layers.39.mlp.experts.122.up_proj", "model.layers.39.mlp.experts.123.up_proj", "model.layers.39.mlp.experts.124.up_proj", "model.layers.39.mlp.experts.125.up_proj", "model.layers.39.mlp.experts.126.up_proj", "model.layers.39.mlp.experts.127.up_proj", "model.layers.39.mlp.experts.128.up_proj", "model.layers.39.mlp.experts.129.up_proj", "model.layers.39.mlp.experts.130.up_proj", "model.layers.39.mlp.experts.131.up_proj", "model.layers.39.mlp.experts.132.up_proj", "model.layers.39.mlp.experts.133.up_proj", "model.layers.39.mlp.experts.134.up_proj", "model.layers.39.mlp.experts.135.up_proj", "model.layers.39.mlp.experts.136.up_proj", "model.layers.39.mlp.experts.137.up_proj", "model.layers.39.mlp.experts.138.up_proj", "model.layers.39.mlp.experts.139.up_proj", "model.layers.39.mlp.experts.140.up_proj", "model.layers.39.mlp.experts.141.up_proj", "model.layers.39.mlp.experts.142.up_proj", "model.layers.39.mlp.experts.143.up_proj", "model.layers.39.mlp.experts.144.up_proj", "model.layers.39.mlp.experts.145.up_proj", "model.layers.39.mlp.experts.146.up_proj", "model.layers.39.mlp.experts.147.up_proj", "model.layers.39.mlp.experts.148.up_proj", "model.layers.39.mlp.experts.149.up_proj", "model.layers.39.mlp.experts.150.up_proj", "model.layers.39.mlp.experts.151.up_proj", "model.layers.39.mlp.experts.152.up_proj", "model.layers.39.mlp.experts.153.up_proj", "model.layers.39.mlp.experts.154.up_proj", "model.layers.39.mlp.experts.155.up_proj", "model.layers.39.mlp.experts.156.up_proj", "model.layers.39.mlp.experts.157.up_proj", "model.layers.39.mlp.experts.158.up_proj", "model.layers.39.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 6.8718276452274e-06, "dbits": 5033164800 } ] }, { "idx": 236, "layers": [ "model.layers.39.mlp.experts.0.down_proj", "model.layers.39.mlp.experts.1.down_proj", "model.layers.39.mlp.experts.2.down_proj", "model.layers.39.mlp.experts.3.down_proj", "model.layers.39.mlp.experts.4.down_proj", "model.layers.39.mlp.experts.5.down_proj", "model.layers.39.mlp.experts.6.down_proj", "model.layers.39.mlp.experts.7.down_proj", "model.layers.39.mlp.experts.8.down_proj", "model.layers.39.mlp.experts.9.down_proj", "model.layers.39.mlp.experts.10.down_proj", "model.layers.39.mlp.experts.11.down_proj", "model.layers.39.mlp.experts.12.down_proj", "model.layers.39.mlp.experts.13.down_proj", "model.layers.39.mlp.experts.14.down_proj", "model.layers.39.mlp.experts.15.down_proj", "model.layers.39.mlp.experts.16.down_proj", "model.layers.39.mlp.experts.17.down_proj", "model.layers.39.mlp.experts.18.down_proj", "model.layers.39.mlp.experts.19.down_proj", "model.layers.39.mlp.experts.20.down_proj", "model.layers.39.mlp.experts.21.down_proj", "model.layers.39.mlp.experts.22.down_proj", "model.layers.39.mlp.experts.23.down_proj", "model.layers.39.mlp.experts.24.down_proj", "model.layers.39.mlp.experts.25.down_proj", "model.layers.39.mlp.experts.26.down_proj", "model.layers.39.mlp.experts.27.down_proj", "model.layers.39.mlp.experts.28.down_proj", "model.layers.39.mlp.experts.29.down_proj", "model.layers.39.mlp.experts.30.down_proj", "model.layers.39.mlp.experts.31.down_proj", "model.layers.39.mlp.experts.32.down_proj", "model.layers.39.mlp.experts.33.down_proj", "model.layers.39.mlp.experts.34.down_proj", "model.layers.39.mlp.experts.35.down_proj", "model.layers.39.mlp.experts.36.down_proj", "model.layers.39.mlp.experts.37.down_proj", "model.layers.39.mlp.experts.38.down_proj", "model.layers.39.mlp.experts.39.down_proj", "model.layers.39.mlp.experts.40.down_proj", "model.layers.39.mlp.experts.41.down_proj", "model.layers.39.mlp.experts.42.down_proj", "model.layers.39.mlp.experts.43.down_proj", "model.layers.39.mlp.experts.44.down_proj", "model.layers.39.mlp.experts.45.down_proj", "model.layers.39.mlp.experts.46.down_proj", "model.layers.39.mlp.experts.47.down_proj", "model.layers.39.mlp.experts.48.down_proj", "model.layers.39.mlp.experts.49.down_proj", "model.layers.39.mlp.experts.50.down_proj", "model.layers.39.mlp.experts.51.down_proj", "model.layers.39.mlp.experts.52.down_proj", "model.layers.39.mlp.experts.53.down_proj", "model.layers.39.mlp.experts.54.down_proj", "model.layers.39.mlp.experts.55.down_proj", "model.layers.39.mlp.experts.56.down_proj", "model.layers.39.mlp.experts.57.down_proj", "model.layers.39.mlp.experts.58.down_proj", "model.layers.39.mlp.experts.59.down_proj", "model.layers.39.mlp.experts.60.down_proj", "model.layers.39.mlp.experts.61.down_proj", "model.layers.39.mlp.experts.62.down_proj", "model.layers.39.mlp.experts.63.down_proj", "model.layers.39.mlp.experts.64.down_proj", "model.layers.39.mlp.experts.65.down_proj", "model.layers.39.mlp.experts.66.down_proj", "model.layers.39.mlp.experts.67.down_proj", "model.layers.39.mlp.experts.68.down_proj", "model.layers.39.mlp.experts.69.down_proj", "model.layers.39.mlp.experts.70.down_proj", "model.layers.39.mlp.experts.71.down_proj", "model.layers.39.mlp.experts.72.down_proj", "model.layers.39.mlp.experts.73.down_proj", "model.layers.39.mlp.experts.74.down_proj", "model.layers.39.mlp.experts.75.down_proj", "model.layers.39.mlp.experts.76.down_proj", "model.layers.39.mlp.experts.77.down_proj", "model.layers.39.mlp.experts.78.down_proj", "model.layers.39.mlp.experts.79.down_proj", "model.layers.39.mlp.experts.80.down_proj", "model.layers.39.mlp.experts.81.down_proj", "model.layers.39.mlp.experts.82.down_proj", "model.layers.39.mlp.experts.83.down_proj", "model.layers.39.mlp.experts.84.down_proj", "model.layers.39.mlp.experts.85.down_proj", "model.layers.39.mlp.experts.86.down_proj", "model.layers.39.mlp.experts.87.down_proj", "model.layers.39.mlp.experts.88.down_proj", "model.layers.39.mlp.experts.89.down_proj", "model.layers.39.mlp.experts.90.down_proj", "model.layers.39.mlp.experts.91.down_proj", "model.layers.39.mlp.experts.92.down_proj", "model.layers.39.mlp.experts.93.down_proj", "model.layers.39.mlp.experts.94.down_proj", "model.layers.39.mlp.experts.95.down_proj", "model.layers.39.mlp.experts.96.down_proj", "model.layers.39.mlp.experts.97.down_proj", "model.layers.39.mlp.experts.98.down_proj", "model.layers.39.mlp.experts.99.down_proj", "model.layers.39.mlp.experts.100.down_proj", "model.layers.39.mlp.experts.101.down_proj", "model.layers.39.mlp.experts.102.down_proj", "model.layers.39.mlp.experts.103.down_proj", "model.layers.39.mlp.experts.104.down_proj", "model.layers.39.mlp.experts.105.down_proj", "model.layers.39.mlp.experts.106.down_proj", "model.layers.39.mlp.experts.107.down_proj", "model.layers.39.mlp.experts.108.down_proj", "model.layers.39.mlp.experts.109.down_proj", "model.layers.39.mlp.experts.110.down_proj", "model.layers.39.mlp.experts.111.down_proj", "model.layers.39.mlp.experts.112.down_proj", "model.layers.39.mlp.experts.113.down_proj", "model.layers.39.mlp.experts.114.down_proj", "model.layers.39.mlp.experts.115.down_proj", "model.layers.39.mlp.experts.116.down_proj", "model.layers.39.mlp.experts.117.down_proj", "model.layers.39.mlp.experts.118.down_proj", "model.layers.39.mlp.experts.119.down_proj", "model.layers.39.mlp.experts.120.down_proj", "model.layers.39.mlp.experts.121.down_proj", "model.layers.39.mlp.experts.122.down_proj", "model.layers.39.mlp.experts.123.down_proj", "model.layers.39.mlp.experts.124.down_proj", "model.layers.39.mlp.experts.125.down_proj", "model.layers.39.mlp.experts.126.down_proj", "model.layers.39.mlp.experts.127.down_proj", "model.layers.39.mlp.experts.128.down_proj", "model.layers.39.mlp.experts.129.down_proj", "model.layers.39.mlp.experts.130.down_proj", "model.layers.39.mlp.experts.131.down_proj", "model.layers.39.mlp.experts.132.down_proj", "model.layers.39.mlp.experts.133.down_proj", "model.layers.39.mlp.experts.134.down_proj", "model.layers.39.mlp.experts.135.down_proj", "model.layers.39.mlp.experts.136.down_proj", "model.layers.39.mlp.experts.137.down_proj", "model.layers.39.mlp.experts.138.down_proj", "model.layers.39.mlp.experts.139.down_proj", "model.layers.39.mlp.experts.140.down_proj", "model.layers.39.mlp.experts.141.down_proj", "model.layers.39.mlp.experts.142.down_proj", "model.layers.39.mlp.experts.143.down_proj", "model.layers.39.mlp.experts.144.down_proj", "model.layers.39.mlp.experts.145.down_proj", "model.layers.39.mlp.experts.146.down_proj", "model.layers.39.mlp.experts.147.down_proj", "model.layers.39.mlp.experts.148.down_proj", "model.layers.39.mlp.experts.149.down_proj", "model.layers.39.mlp.experts.150.down_proj", "model.layers.39.mlp.experts.151.down_proj", "model.layers.39.mlp.experts.152.down_proj", "model.layers.39.mlp.experts.153.down_proj", "model.layers.39.mlp.experts.154.down_proj", "model.layers.39.mlp.experts.155.down_proj", "model.layers.39.mlp.experts.156.down_proj", "model.layers.39.mlp.experts.157.down_proj", "model.layers.39.mlp.experts.158.down_proj", "model.layers.39.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 7.100426591932687e-06, "dbits": 2516582400 } ] }, { "idx": 237, "layers": [ "model.layers.40.self_attn.q_proj" ], "candidates": [ { "dkld": 3.9957812987263634e-07, "dbits": 125829120 } ] }, { "idx": 238, "layers": [ "model.layers.40.self_attn.k_proj", "model.layers.40.self_attn.v_proj" ], "candidates": [ { "dkld": -5.416988278739176e-06, "dbits": 20971520 } ] }, { "idx": 239, "layers": [ "model.layers.40.self_attn.o_proj" ], "candidates": [ { "dkld": 1.0261195711791299e-06, "dbits": 125829120 } ] }, { "idx": 240, "layers": [ "model.layers.40.mlp.shared_experts.gate_proj", "model.layers.40.mlp.shared_experts.up_proj", "model.layers.40.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.1128769731148644e-06, "dbits": 47185920 } ] }, { "idx": 241, "layers": [ "model.layers.40.mlp.experts.0.gate_proj", "model.layers.40.mlp.experts.1.gate_proj", "model.layers.40.mlp.experts.2.gate_proj", "model.layers.40.mlp.experts.3.gate_proj", "model.layers.40.mlp.experts.4.gate_proj", "model.layers.40.mlp.experts.5.gate_proj", "model.layers.40.mlp.experts.6.gate_proj", "model.layers.40.mlp.experts.7.gate_proj", "model.layers.40.mlp.experts.8.gate_proj", "model.layers.40.mlp.experts.9.gate_proj", "model.layers.40.mlp.experts.10.gate_proj", "model.layers.40.mlp.experts.11.gate_proj", "model.layers.40.mlp.experts.12.gate_proj", "model.layers.40.mlp.experts.13.gate_proj", "model.layers.40.mlp.experts.14.gate_proj", "model.layers.40.mlp.experts.15.gate_proj", "model.layers.40.mlp.experts.16.gate_proj", "model.layers.40.mlp.experts.17.gate_proj", "model.layers.40.mlp.experts.18.gate_proj", "model.layers.40.mlp.experts.19.gate_proj", "model.layers.40.mlp.experts.20.gate_proj", "model.layers.40.mlp.experts.21.gate_proj", "model.layers.40.mlp.experts.22.gate_proj", "model.layers.40.mlp.experts.23.gate_proj", "model.layers.40.mlp.experts.24.gate_proj", "model.layers.40.mlp.experts.25.gate_proj", "model.layers.40.mlp.experts.26.gate_proj", "model.layers.40.mlp.experts.27.gate_proj", "model.layers.40.mlp.experts.28.gate_proj", "model.layers.40.mlp.experts.29.gate_proj", "model.layers.40.mlp.experts.30.gate_proj", "model.layers.40.mlp.experts.31.gate_proj", "model.layers.40.mlp.experts.32.gate_proj", "model.layers.40.mlp.experts.33.gate_proj", "model.layers.40.mlp.experts.34.gate_proj", "model.layers.40.mlp.experts.35.gate_proj", "model.layers.40.mlp.experts.36.gate_proj", "model.layers.40.mlp.experts.37.gate_proj", "model.layers.40.mlp.experts.38.gate_proj", "model.layers.40.mlp.experts.39.gate_proj", "model.layers.40.mlp.experts.40.gate_proj", "model.layers.40.mlp.experts.41.gate_proj", "model.layers.40.mlp.experts.42.gate_proj", "model.layers.40.mlp.experts.43.gate_proj", "model.layers.40.mlp.experts.44.gate_proj", "model.layers.40.mlp.experts.45.gate_proj", "model.layers.40.mlp.experts.46.gate_proj", "model.layers.40.mlp.experts.47.gate_proj", "model.layers.40.mlp.experts.48.gate_proj", "model.layers.40.mlp.experts.49.gate_proj", "model.layers.40.mlp.experts.50.gate_proj", "model.layers.40.mlp.experts.51.gate_proj", "model.layers.40.mlp.experts.52.gate_proj", "model.layers.40.mlp.experts.53.gate_proj", "model.layers.40.mlp.experts.54.gate_proj", "model.layers.40.mlp.experts.55.gate_proj", "model.layers.40.mlp.experts.56.gate_proj", "model.layers.40.mlp.experts.57.gate_proj", "model.layers.40.mlp.experts.58.gate_proj", "model.layers.40.mlp.experts.59.gate_proj", "model.layers.40.mlp.experts.60.gate_proj", "model.layers.40.mlp.experts.61.gate_proj", "model.layers.40.mlp.experts.62.gate_proj", "model.layers.40.mlp.experts.63.gate_proj", "model.layers.40.mlp.experts.64.gate_proj", "model.layers.40.mlp.experts.65.gate_proj", "model.layers.40.mlp.experts.66.gate_proj", "model.layers.40.mlp.experts.67.gate_proj", "model.layers.40.mlp.experts.68.gate_proj", "model.layers.40.mlp.experts.69.gate_proj", "model.layers.40.mlp.experts.70.gate_proj", "model.layers.40.mlp.experts.71.gate_proj", "model.layers.40.mlp.experts.72.gate_proj", "model.layers.40.mlp.experts.73.gate_proj", "model.layers.40.mlp.experts.74.gate_proj", "model.layers.40.mlp.experts.75.gate_proj", "model.layers.40.mlp.experts.76.gate_proj", "model.layers.40.mlp.experts.77.gate_proj", "model.layers.40.mlp.experts.78.gate_proj", "model.layers.40.mlp.experts.79.gate_proj", "model.layers.40.mlp.experts.80.gate_proj", "model.layers.40.mlp.experts.81.gate_proj", "model.layers.40.mlp.experts.82.gate_proj", "model.layers.40.mlp.experts.83.gate_proj", "model.layers.40.mlp.experts.84.gate_proj", "model.layers.40.mlp.experts.85.gate_proj", "model.layers.40.mlp.experts.86.gate_proj", "model.layers.40.mlp.experts.87.gate_proj", "model.layers.40.mlp.experts.88.gate_proj", "model.layers.40.mlp.experts.89.gate_proj", "model.layers.40.mlp.experts.90.gate_proj", "model.layers.40.mlp.experts.91.gate_proj", "model.layers.40.mlp.experts.92.gate_proj", "model.layers.40.mlp.experts.93.gate_proj", "model.layers.40.mlp.experts.94.gate_proj", "model.layers.40.mlp.experts.95.gate_proj", "model.layers.40.mlp.experts.96.gate_proj", "model.layers.40.mlp.experts.97.gate_proj", "model.layers.40.mlp.experts.98.gate_proj", "model.layers.40.mlp.experts.99.gate_proj", "model.layers.40.mlp.experts.100.gate_proj", "model.layers.40.mlp.experts.101.gate_proj", "model.layers.40.mlp.experts.102.gate_proj", "model.layers.40.mlp.experts.103.gate_proj", "model.layers.40.mlp.experts.104.gate_proj", "model.layers.40.mlp.experts.105.gate_proj", "model.layers.40.mlp.experts.106.gate_proj", "model.layers.40.mlp.experts.107.gate_proj", "model.layers.40.mlp.experts.108.gate_proj", "model.layers.40.mlp.experts.109.gate_proj", "model.layers.40.mlp.experts.110.gate_proj", "model.layers.40.mlp.experts.111.gate_proj", "model.layers.40.mlp.experts.112.gate_proj", "model.layers.40.mlp.experts.113.gate_proj", "model.layers.40.mlp.experts.114.gate_proj", "model.layers.40.mlp.experts.115.gate_proj", "model.layers.40.mlp.experts.116.gate_proj", "model.layers.40.mlp.experts.117.gate_proj", "model.layers.40.mlp.experts.118.gate_proj", "model.layers.40.mlp.experts.119.gate_proj", "model.layers.40.mlp.experts.120.gate_proj", "model.layers.40.mlp.experts.121.gate_proj", "model.layers.40.mlp.experts.122.gate_proj", "model.layers.40.mlp.experts.123.gate_proj", "model.layers.40.mlp.experts.124.gate_proj", "model.layers.40.mlp.experts.125.gate_proj", "model.layers.40.mlp.experts.126.gate_proj", "model.layers.40.mlp.experts.127.gate_proj", "model.layers.40.mlp.experts.128.gate_proj", "model.layers.40.mlp.experts.129.gate_proj", "model.layers.40.mlp.experts.130.gate_proj", "model.layers.40.mlp.experts.131.gate_proj", "model.layers.40.mlp.experts.132.gate_proj", "model.layers.40.mlp.experts.133.gate_proj", "model.layers.40.mlp.experts.134.gate_proj", "model.layers.40.mlp.experts.135.gate_proj", "model.layers.40.mlp.experts.136.gate_proj", "model.layers.40.mlp.experts.137.gate_proj", "model.layers.40.mlp.experts.138.gate_proj", "model.layers.40.mlp.experts.139.gate_proj", "model.layers.40.mlp.experts.140.gate_proj", "model.layers.40.mlp.experts.141.gate_proj", "model.layers.40.mlp.experts.142.gate_proj", "model.layers.40.mlp.experts.143.gate_proj", "model.layers.40.mlp.experts.144.gate_proj", "model.layers.40.mlp.experts.145.gate_proj", "model.layers.40.mlp.experts.146.gate_proj", "model.layers.40.mlp.experts.147.gate_proj", "model.layers.40.mlp.experts.148.gate_proj", "model.layers.40.mlp.experts.149.gate_proj", "model.layers.40.mlp.experts.150.gate_proj", "model.layers.40.mlp.experts.151.gate_proj", "model.layers.40.mlp.experts.152.gate_proj", "model.layers.40.mlp.experts.153.gate_proj", "model.layers.40.mlp.experts.154.gate_proj", "model.layers.40.mlp.experts.155.gate_proj", "model.layers.40.mlp.experts.156.gate_proj", "model.layers.40.mlp.experts.157.gate_proj", "model.layers.40.mlp.experts.158.gate_proj", "model.layers.40.mlp.experts.159.gate_proj", "model.layers.40.mlp.experts.0.up_proj", "model.layers.40.mlp.experts.1.up_proj", "model.layers.40.mlp.experts.2.up_proj", "model.layers.40.mlp.experts.3.up_proj", "model.layers.40.mlp.experts.4.up_proj", "model.layers.40.mlp.experts.5.up_proj", "model.layers.40.mlp.experts.6.up_proj", "model.layers.40.mlp.experts.7.up_proj", "model.layers.40.mlp.experts.8.up_proj", "model.layers.40.mlp.experts.9.up_proj", "model.layers.40.mlp.experts.10.up_proj", "model.layers.40.mlp.experts.11.up_proj", "model.layers.40.mlp.experts.12.up_proj", "model.layers.40.mlp.experts.13.up_proj", "model.layers.40.mlp.experts.14.up_proj", "model.layers.40.mlp.experts.15.up_proj", "model.layers.40.mlp.experts.16.up_proj", "model.layers.40.mlp.experts.17.up_proj", "model.layers.40.mlp.experts.18.up_proj", "model.layers.40.mlp.experts.19.up_proj", "model.layers.40.mlp.experts.20.up_proj", "model.layers.40.mlp.experts.21.up_proj", "model.layers.40.mlp.experts.22.up_proj", "model.layers.40.mlp.experts.23.up_proj", "model.layers.40.mlp.experts.24.up_proj", "model.layers.40.mlp.experts.25.up_proj", "model.layers.40.mlp.experts.26.up_proj", "model.layers.40.mlp.experts.27.up_proj", "model.layers.40.mlp.experts.28.up_proj", "model.layers.40.mlp.experts.29.up_proj", "model.layers.40.mlp.experts.30.up_proj", "model.layers.40.mlp.experts.31.up_proj", "model.layers.40.mlp.experts.32.up_proj", "model.layers.40.mlp.experts.33.up_proj", "model.layers.40.mlp.experts.34.up_proj", "model.layers.40.mlp.experts.35.up_proj", "model.layers.40.mlp.experts.36.up_proj", "model.layers.40.mlp.experts.37.up_proj", "model.layers.40.mlp.experts.38.up_proj", "model.layers.40.mlp.experts.39.up_proj", "model.layers.40.mlp.experts.40.up_proj", "model.layers.40.mlp.experts.41.up_proj", "model.layers.40.mlp.experts.42.up_proj", "model.layers.40.mlp.experts.43.up_proj", "model.layers.40.mlp.experts.44.up_proj", "model.layers.40.mlp.experts.45.up_proj", "model.layers.40.mlp.experts.46.up_proj", "model.layers.40.mlp.experts.47.up_proj", "model.layers.40.mlp.experts.48.up_proj", "model.layers.40.mlp.experts.49.up_proj", "model.layers.40.mlp.experts.50.up_proj", "model.layers.40.mlp.experts.51.up_proj", "model.layers.40.mlp.experts.52.up_proj", "model.layers.40.mlp.experts.53.up_proj", "model.layers.40.mlp.experts.54.up_proj", "model.layers.40.mlp.experts.55.up_proj", "model.layers.40.mlp.experts.56.up_proj", "model.layers.40.mlp.experts.57.up_proj", "model.layers.40.mlp.experts.58.up_proj", "model.layers.40.mlp.experts.59.up_proj", "model.layers.40.mlp.experts.60.up_proj", "model.layers.40.mlp.experts.61.up_proj", "model.layers.40.mlp.experts.62.up_proj", "model.layers.40.mlp.experts.63.up_proj", "model.layers.40.mlp.experts.64.up_proj", "model.layers.40.mlp.experts.65.up_proj", "model.layers.40.mlp.experts.66.up_proj", "model.layers.40.mlp.experts.67.up_proj", "model.layers.40.mlp.experts.68.up_proj", "model.layers.40.mlp.experts.69.up_proj", "model.layers.40.mlp.experts.70.up_proj", "model.layers.40.mlp.experts.71.up_proj", "model.layers.40.mlp.experts.72.up_proj", "model.layers.40.mlp.experts.73.up_proj", "model.layers.40.mlp.experts.74.up_proj", "model.layers.40.mlp.experts.75.up_proj", "model.layers.40.mlp.experts.76.up_proj", "model.layers.40.mlp.experts.77.up_proj", "model.layers.40.mlp.experts.78.up_proj", "model.layers.40.mlp.experts.79.up_proj", "model.layers.40.mlp.experts.80.up_proj", "model.layers.40.mlp.experts.81.up_proj", "model.layers.40.mlp.experts.82.up_proj", "model.layers.40.mlp.experts.83.up_proj", "model.layers.40.mlp.experts.84.up_proj", "model.layers.40.mlp.experts.85.up_proj", "model.layers.40.mlp.experts.86.up_proj", "model.layers.40.mlp.experts.87.up_proj", "model.layers.40.mlp.experts.88.up_proj", "model.layers.40.mlp.experts.89.up_proj", "model.layers.40.mlp.experts.90.up_proj", "model.layers.40.mlp.experts.91.up_proj", "model.layers.40.mlp.experts.92.up_proj", "model.layers.40.mlp.experts.93.up_proj", "model.layers.40.mlp.experts.94.up_proj", "model.layers.40.mlp.experts.95.up_proj", "model.layers.40.mlp.experts.96.up_proj", "model.layers.40.mlp.experts.97.up_proj", "model.layers.40.mlp.experts.98.up_proj", "model.layers.40.mlp.experts.99.up_proj", "model.layers.40.mlp.experts.100.up_proj", "model.layers.40.mlp.experts.101.up_proj", "model.layers.40.mlp.experts.102.up_proj", "model.layers.40.mlp.experts.103.up_proj", "model.layers.40.mlp.experts.104.up_proj", "model.layers.40.mlp.experts.105.up_proj", "model.layers.40.mlp.experts.106.up_proj", "model.layers.40.mlp.experts.107.up_proj", "model.layers.40.mlp.experts.108.up_proj", "model.layers.40.mlp.experts.109.up_proj", "model.layers.40.mlp.experts.110.up_proj", "model.layers.40.mlp.experts.111.up_proj", "model.layers.40.mlp.experts.112.up_proj", "model.layers.40.mlp.experts.113.up_proj", "model.layers.40.mlp.experts.114.up_proj", "model.layers.40.mlp.experts.115.up_proj", "model.layers.40.mlp.experts.116.up_proj", "model.layers.40.mlp.experts.117.up_proj", "model.layers.40.mlp.experts.118.up_proj", "model.layers.40.mlp.experts.119.up_proj", "model.layers.40.mlp.experts.120.up_proj", "model.layers.40.mlp.experts.121.up_proj", "model.layers.40.mlp.experts.122.up_proj", "model.layers.40.mlp.experts.123.up_proj", "model.layers.40.mlp.experts.124.up_proj", "model.layers.40.mlp.experts.125.up_proj", "model.layers.40.mlp.experts.126.up_proj", "model.layers.40.mlp.experts.127.up_proj", "model.layers.40.mlp.experts.128.up_proj", "model.layers.40.mlp.experts.129.up_proj", "model.layers.40.mlp.experts.130.up_proj", "model.layers.40.mlp.experts.131.up_proj", "model.layers.40.mlp.experts.132.up_proj", "model.layers.40.mlp.experts.133.up_proj", "model.layers.40.mlp.experts.134.up_proj", "model.layers.40.mlp.experts.135.up_proj", "model.layers.40.mlp.experts.136.up_proj", "model.layers.40.mlp.experts.137.up_proj", "model.layers.40.mlp.experts.138.up_proj", "model.layers.40.mlp.experts.139.up_proj", "model.layers.40.mlp.experts.140.up_proj", "model.layers.40.mlp.experts.141.up_proj", "model.layers.40.mlp.experts.142.up_proj", "model.layers.40.mlp.experts.143.up_proj", "model.layers.40.mlp.experts.144.up_proj", "model.layers.40.mlp.experts.145.up_proj", "model.layers.40.mlp.experts.146.up_proj", "model.layers.40.mlp.experts.147.up_proj", "model.layers.40.mlp.experts.148.up_proj", "model.layers.40.mlp.experts.149.up_proj", "model.layers.40.mlp.experts.150.up_proj", "model.layers.40.mlp.experts.151.up_proj", "model.layers.40.mlp.experts.152.up_proj", "model.layers.40.mlp.experts.153.up_proj", "model.layers.40.mlp.experts.154.up_proj", "model.layers.40.mlp.experts.155.up_proj", "model.layers.40.mlp.experts.156.up_proj", "model.layers.40.mlp.experts.157.up_proj", "model.layers.40.mlp.experts.158.up_proj", "model.layers.40.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.653271076269426e-06, "dbits": 5033164800 } ] }, { "idx": 242, "layers": [ "model.layers.40.mlp.experts.0.down_proj", "model.layers.40.mlp.experts.1.down_proj", "model.layers.40.mlp.experts.2.down_proj", "model.layers.40.mlp.experts.3.down_proj", "model.layers.40.mlp.experts.4.down_proj", "model.layers.40.mlp.experts.5.down_proj", "model.layers.40.mlp.experts.6.down_proj", "model.layers.40.mlp.experts.7.down_proj", "model.layers.40.mlp.experts.8.down_proj", "model.layers.40.mlp.experts.9.down_proj", "model.layers.40.mlp.experts.10.down_proj", "model.layers.40.mlp.experts.11.down_proj", "model.layers.40.mlp.experts.12.down_proj", "model.layers.40.mlp.experts.13.down_proj", "model.layers.40.mlp.experts.14.down_proj", "model.layers.40.mlp.experts.15.down_proj", "model.layers.40.mlp.experts.16.down_proj", "model.layers.40.mlp.experts.17.down_proj", "model.layers.40.mlp.experts.18.down_proj", "model.layers.40.mlp.experts.19.down_proj", "model.layers.40.mlp.experts.20.down_proj", "model.layers.40.mlp.experts.21.down_proj", "model.layers.40.mlp.experts.22.down_proj", "model.layers.40.mlp.experts.23.down_proj", "model.layers.40.mlp.experts.24.down_proj", "model.layers.40.mlp.experts.25.down_proj", "model.layers.40.mlp.experts.26.down_proj", "model.layers.40.mlp.experts.27.down_proj", "model.layers.40.mlp.experts.28.down_proj", "model.layers.40.mlp.experts.29.down_proj", "model.layers.40.mlp.experts.30.down_proj", "model.layers.40.mlp.experts.31.down_proj", "model.layers.40.mlp.experts.32.down_proj", "model.layers.40.mlp.experts.33.down_proj", "model.layers.40.mlp.experts.34.down_proj", "model.layers.40.mlp.experts.35.down_proj", "model.layers.40.mlp.experts.36.down_proj", "model.layers.40.mlp.experts.37.down_proj", "model.layers.40.mlp.experts.38.down_proj", "model.layers.40.mlp.experts.39.down_proj", "model.layers.40.mlp.experts.40.down_proj", "model.layers.40.mlp.experts.41.down_proj", "model.layers.40.mlp.experts.42.down_proj", "model.layers.40.mlp.experts.43.down_proj", "model.layers.40.mlp.experts.44.down_proj", "model.layers.40.mlp.experts.45.down_proj", "model.layers.40.mlp.experts.46.down_proj", "model.layers.40.mlp.experts.47.down_proj", "model.layers.40.mlp.experts.48.down_proj", "model.layers.40.mlp.experts.49.down_proj", "model.layers.40.mlp.experts.50.down_proj", "model.layers.40.mlp.experts.51.down_proj", "model.layers.40.mlp.experts.52.down_proj", "model.layers.40.mlp.experts.53.down_proj", "model.layers.40.mlp.experts.54.down_proj", "model.layers.40.mlp.experts.55.down_proj", "model.layers.40.mlp.experts.56.down_proj", "model.layers.40.mlp.experts.57.down_proj", "model.layers.40.mlp.experts.58.down_proj", "model.layers.40.mlp.experts.59.down_proj", "model.layers.40.mlp.experts.60.down_proj", "model.layers.40.mlp.experts.61.down_proj", "model.layers.40.mlp.experts.62.down_proj", "model.layers.40.mlp.experts.63.down_proj", "model.layers.40.mlp.experts.64.down_proj", "model.layers.40.mlp.experts.65.down_proj", "model.layers.40.mlp.experts.66.down_proj", "model.layers.40.mlp.experts.67.down_proj", "model.layers.40.mlp.experts.68.down_proj", "model.layers.40.mlp.experts.69.down_proj", "model.layers.40.mlp.experts.70.down_proj", "model.layers.40.mlp.experts.71.down_proj", "model.layers.40.mlp.experts.72.down_proj", "model.layers.40.mlp.experts.73.down_proj", "model.layers.40.mlp.experts.74.down_proj", "model.layers.40.mlp.experts.75.down_proj", "model.layers.40.mlp.experts.76.down_proj", "model.layers.40.mlp.experts.77.down_proj", "model.layers.40.mlp.experts.78.down_proj", "model.layers.40.mlp.experts.79.down_proj", "model.layers.40.mlp.experts.80.down_proj", "model.layers.40.mlp.experts.81.down_proj", "model.layers.40.mlp.experts.82.down_proj", "model.layers.40.mlp.experts.83.down_proj", "model.layers.40.mlp.experts.84.down_proj", "model.layers.40.mlp.experts.85.down_proj", "model.layers.40.mlp.experts.86.down_proj", "model.layers.40.mlp.experts.87.down_proj", "model.layers.40.mlp.experts.88.down_proj", "model.layers.40.mlp.experts.89.down_proj", "model.layers.40.mlp.experts.90.down_proj", "model.layers.40.mlp.experts.91.down_proj", "model.layers.40.mlp.experts.92.down_proj", "model.layers.40.mlp.experts.93.down_proj", "model.layers.40.mlp.experts.94.down_proj", "model.layers.40.mlp.experts.95.down_proj", "model.layers.40.mlp.experts.96.down_proj", "model.layers.40.mlp.experts.97.down_proj", "model.layers.40.mlp.experts.98.down_proj", "model.layers.40.mlp.experts.99.down_proj", "model.layers.40.mlp.experts.100.down_proj", "model.layers.40.mlp.experts.101.down_proj", "model.layers.40.mlp.experts.102.down_proj", "model.layers.40.mlp.experts.103.down_proj", "model.layers.40.mlp.experts.104.down_proj", "model.layers.40.mlp.experts.105.down_proj", "model.layers.40.mlp.experts.106.down_proj", "model.layers.40.mlp.experts.107.down_proj", "model.layers.40.mlp.experts.108.down_proj", "model.layers.40.mlp.experts.109.down_proj", "model.layers.40.mlp.experts.110.down_proj", "model.layers.40.mlp.experts.111.down_proj", "model.layers.40.mlp.experts.112.down_proj", "model.layers.40.mlp.experts.113.down_proj", "model.layers.40.mlp.experts.114.down_proj", "model.layers.40.mlp.experts.115.down_proj", "model.layers.40.mlp.experts.116.down_proj", "model.layers.40.mlp.experts.117.down_proj", "model.layers.40.mlp.experts.118.down_proj", "model.layers.40.mlp.experts.119.down_proj", "model.layers.40.mlp.experts.120.down_proj", "model.layers.40.mlp.experts.121.down_proj", "model.layers.40.mlp.experts.122.down_proj", "model.layers.40.mlp.experts.123.down_proj", "model.layers.40.mlp.experts.124.down_proj", "model.layers.40.mlp.experts.125.down_proj", "model.layers.40.mlp.experts.126.down_proj", "model.layers.40.mlp.experts.127.down_proj", "model.layers.40.mlp.experts.128.down_proj", "model.layers.40.mlp.experts.129.down_proj", "model.layers.40.mlp.experts.130.down_proj", "model.layers.40.mlp.experts.131.down_proj", "model.layers.40.mlp.experts.132.down_proj", "model.layers.40.mlp.experts.133.down_proj", "model.layers.40.mlp.experts.134.down_proj", "model.layers.40.mlp.experts.135.down_proj", "model.layers.40.mlp.experts.136.down_proj", "model.layers.40.mlp.experts.137.down_proj", "model.layers.40.mlp.experts.138.down_proj", "model.layers.40.mlp.experts.139.down_proj", "model.layers.40.mlp.experts.140.down_proj", "model.layers.40.mlp.experts.141.down_proj", "model.layers.40.mlp.experts.142.down_proj", "model.layers.40.mlp.experts.143.down_proj", "model.layers.40.mlp.experts.144.down_proj", "model.layers.40.mlp.experts.145.down_proj", "model.layers.40.mlp.experts.146.down_proj", "model.layers.40.mlp.experts.147.down_proj", "model.layers.40.mlp.experts.148.down_proj", "model.layers.40.mlp.experts.149.down_proj", "model.layers.40.mlp.experts.150.down_proj", "model.layers.40.mlp.experts.151.down_proj", "model.layers.40.mlp.experts.152.down_proj", "model.layers.40.mlp.experts.153.down_proj", "model.layers.40.mlp.experts.154.down_proj", "model.layers.40.mlp.experts.155.down_proj", "model.layers.40.mlp.experts.156.down_proj", "model.layers.40.mlp.experts.157.down_proj", "model.layers.40.mlp.experts.158.down_proj", "model.layers.40.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.477128873579204e-06, "dbits": 2516582400 } ] }, { "idx": 243, "layers": [ "model.layers.41.self_attn.q_proj" ], "candidates": [ { "dkld": 1.3006880180910447e-06, "dbits": 125829120 } ] }, { "idx": 244, "layers": [ "model.layers.41.self_attn.k_proj", "model.layers.41.self_attn.v_proj" ], "candidates": [ { "dkld": -2.479768591001608e-06, "dbits": 20971520 } ] }, { "idx": 245, "layers": [ "model.layers.41.self_attn.o_proj" ], "candidates": [ { "dkld": 4.3968291720374985e-06, "dbits": 125829120 } ] }, { "idx": 246, "layers": [ "model.layers.41.mlp.shared_experts.gate_proj", "model.layers.41.mlp.shared_experts.up_proj", "model.layers.41.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.201094710268102e-06, "dbits": 47185920 } ] }, { "idx": 247, "layers": [ "model.layers.41.mlp.experts.0.gate_proj", "model.layers.41.mlp.experts.1.gate_proj", "model.layers.41.mlp.experts.2.gate_proj", "model.layers.41.mlp.experts.3.gate_proj", "model.layers.41.mlp.experts.4.gate_proj", "model.layers.41.mlp.experts.5.gate_proj", "model.layers.41.mlp.experts.6.gate_proj", "model.layers.41.mlp.experts.7.gate_proj", "model.layers.41.mlp.experts.8.gate_proj", "model.layers.41.mlp.experts.9.gate_proj", "model.layers.41.mlp.experts.10.gate_proj", "model.layers.41.mlp.experts.11.gate_proj", "model.layers.41.mlp.experts.12.gate_proj", "model.layers.41.mlp.experts.13.gate_proj", "model.layers.41.mlp.experts.14.gate_proj", "model.layers.41.mlp.experts.15.gate_proj", "model.layers.41.mlp.experts.16.gate_proj", "model.layers.41.mlp.experts.17.gate_proj", "model.layers.41.mlp.experts.18.gate_proj", "model.layers.41.mlp.experts.19.gate_proj", "model.layers.41.mlp.experts.20.gate_proj", "model.layers.41.mlp.experts.21.gate_proj", "model.layers.41.mlp.experts.22.gate_proj", "model.layers.41.mlp.experts.23.gate_proj", "model.layers.41.mlp.experts.24.gate_proj", "model.layers.41.mlp.experts.25.gate_proj", "model.layers.41.mlp.experts.26.gate_proj", "model.layers.41.mlp.experts.27.gate_proj", "model.layers.41.mlp.experts.28.gate_proj", "model.layers.41.mlp.experts.29.gate_proj", "model.layers.41.mlp.experts.30.gate_proj", "model.layers.41.mlp.experts.31.gate_proj", "model.layers.41.mlp.experts.32.gate_proj", "model.layers.41.mlp.experts.33.gate_proj", "model.layers.41.mlp.experts.34.gate_proj", "model.layers.41.mlp.experts.35.gate_proj", "model.layers.41.mlp.experts.36.gate_proj", "model.layers.41.mlp.experts.37.gate_proj", "model.layers.41.mlp.experts.38.gate_proj", "model.layers.41.mlp.experts.39.gate_proj", "model.layers.41.mlp.experts.40.gate_proj", "model.layers.41.mlp.experts.41.gate_proj", "model.layers.41.mlp.experts.42.gate_proj", "model.layers.41.mlp.experts.43.gate_proj", "model.layers.41.mlp.experts.44.gate_proj", "model.layers.41.mlp.experts.45.gate_proj", "model.layers.41.mlp.experts.46.gate_proj", "model.layers.41.mlp.experts.47.gate_proj", "model.layers.41.mlp.experts.48.gate_proj", "model.layers.41.mlp.experts.49.gate_proj", "model.layers.41.mlp.experts.50.gate_proj", "model.layers.41.mlp.experts.51.gate_proj", "model.layers.41.mlp.experts.52.gate_proj", "model.layers.41.mlp.experts.53.gate_proj", "model.layers.41.mlp.experts.54.gate_proj", "model.layers.41.mlp.experts.55.gate_proj", "model.layers.41.mlp.experts.56.gate_proj", "model.layers.41.mlp.experts.57.gate_proj", "model.layers.41.mlp.experts.58.gate_proj", "model.layers.41.mlp.experts.59.gate_proj", "model.layers.41.mlp.experts.60.gate_proj", "model.layers.41.mlp.experts.61.gate_proj", "model.layers.41.mlp.experts.62.gate_proj", "model.layers.41.mlp.experts.63.gate_proj", "model.layers.41.mlp.experts.64.gate_proj", "model.layers.41.mlp.experts.65.gate_proj", "model.layers.41.mlp.experts.66.gate_proj", "model.layers.41.mlp.experts.67.gate_proj", "model.layers.41.mlp.experts.68.gate_proj", "model.layers.41.mlp.experts.69.gate_proj", "model.layers.41.mlp.experts.70.gate_proj", "model.layers.41.mlp.experts.71.gate_proj", "model.layers.41.mlp.experts.72.gate_proj", "model.layers.41.mlp.experts.73.gate_proj", "model.layers.41.mlp.experts.74.gate_proj", "model.layers.41.mlp.experts.75.gate_proj", "model.layers.41.mlp.experts.76.gate_proj", "model.layers.41.mlp.experts.77.gate_proj", "model.layers.41.mlp.experts.78.gate_proj", "model.layers.41.mlp.experts.79.gate_proj", "model.layers.41.mlp.experts.80.gate_proj", "model.layers.41.mlp.experts.81.gate_proj", "model.layers.41.mlp.experts.82.gate_proj", "model.layers.41.mlp.experts.83.gate_proj", "model.layers.41.mlp.experts.84.gate_proj", "model.layers.41.mlp.experts.85.gate_proj", "model.layers.41.mlp.experts.86.gate_proj", "model.layers.41.mlp.experts.87.gate_proj", "model.layers.41.mlp.experts.88.gate_proj", "model.layers.41.mlp.experts.89.gate_proj", "model.layers.41.mlp.experts.90.gate_proj", "model.layers.41.mlp.experts.91.gate_proj", "model.layers.41.mlp.experts.92.gate_proj", "model.layers.41.mlp.experts.93.gate_proj", "model.layers.41.mlp.experts.94.gate_proj", "model.layers.41.mlp.experts.95.gate_proj", "model.layers.41.mlp.experts.96.gate_proj", "model.layers.41.mlp.experts.97.gate_proj", "model.layers.41.mlp.experts.98.gate_proj", "model.layers.41.mlp.experts.99.gate_proj", "model.layers.41.mlp.experts.100.gate_proj", "model.layers.41.mlp.experts.101.gate_proj", "model.layers.41.mlp.experts.102.gate_proj", "model.layers.41.mlp.experts.103.gate_proj", "model.layers.41.mlp.experts.104.gate_proj", "model.layers.41.mlp.experts.105.gate_proj", "model.layers.41.mlp.experts.106.gate_proj", "model.layers.41.mlp.experts.107.gate_proj", "model.layers.41.mlp.experts.108.gate_proj", "model.layers.41.mlp.experts.109.gate_proj", "model.layers.41.mlp.experts.110.gate_proj", "model.layers.41.mlp.experts.111.gate_proj", "model.layers.41.mlp.experts.112.gate_proj", "model.layers.41.mlp.experts.113.gate_proj", "model.layers.41.mlp.experts.114.gate_proj", "model.layers.41.mlp.experts.115.gate_proj", "model.layers.41.mlp.experts.116.gate_proj", "model.layers.41.mlp.experts.117.gate_proj", "model.layers.41.mlp.experts.118.gate_proj", "model.layers.41.mlp.experts.119.gate_proj", "model.layers.41.mlp.experts.120.gate_proj", "model.layers.41.mlp.experts.121.gate_proj", "model.layers.41.mlp.experts.122.gate_proj", "model.layers.41.mlp.experts.123.gate_proj", "model.layers.41.mlp.experts.124.gate_proj", "model.layers.41.mlp.experts.125.gate_proj", "model.layers.41.mlp.experts.126.gate_proj", "model.layers.41.mlp.experts.127.gate_proj", "model.layers.41.mlp.experts.128.gate_proj", "model.layers.41.mlp.experts.129.gate_proj", "model.layers.41.mlp.experts.130.gate_proj", "model.layers.41.mlp.experts.131.gate_proj", "model.layers.41.mlp.experts.132.gate_proj", "model.layers.41.mlp.experts.133.gate_proj", "model.layers.41.mlp.experts.134.gate_proj", "model.layers.41.mlp.experts.135.gate_proj", "model.layers.41.mlp.experts.136.gate_proj", "model.layers.41.mlp.experts.137.gate_proj", "model.layers.41.mlp.experts.138.gate_proj", "model.layers.41.mlp.experts.139.gate_proj", "model.layers.41.mlp.experts.140.gate_proj", "model.layers.41.mlp.experts.141.gate_proj", "model.layers.41.mlp.experts.142.gate_proj", "model.layers.41.mlp.experts.143.gate_proj", "model.layers.41.mlp.experts.144.gate_proj", "model.layers.41.mlp.experts.145.gate_proj", "model.layers.41.mlp.experts.146.gate_proj", "model.layers.41.mlp.experts.147.gate_proj", "model.layers.41.mlp.experts.148.gate_proj", "model.layers.41.mlp.experts.149.gate_proj", "model.layers.41.mlp.experts.150.gate_proj", "model.layers.41.mlp.experts.151.gate_proj", "model.layers.41.mlp.experts.152.gate_proj", "model.layers.41.mlp.experts.153.gate_proj", "model.layers.41.mlp.experts.154.gate_proj", "model.layers.41.mlp.experts.155.gate_proj", "model.layers.41.mlp.experts.156.gate_proj", "model.layers.41.mlp.experts.157.gate_proj", "model.layers.41.mlp.experts.158.gate_proj", "model.layers.41.mlp.experts.159.gate_proj", "model.layers.41.mlp.experts.0.up_proj", "model.layers.41.mlp.experts.1.up_proj", "model.layers.41.mlp.experts.2.up_proj", "model.layers.41.mlp.experts.3.up_proj", "model.layers.41.mlp.experts.4.up_proj", "model.layers.41.mlp.experts.5.up_proj", "model.layers.41.mlp.experts.6.up_proj", "model.layers.41.mlp.experts.7.up_proj", "model.layers.41.mlp.experts.8.up_proj", "model.layers.41.mlp.experts.9.up_proj", "model.layers.41.mlp.experts.10.up_proj", "model.layers.41.mlp.experts.11.up_proj", "model.layers.41.mlp.experts.12.up_proj", "model.layers.41.mlp.experts.13.up_proj", "model.layers.41.mlp.experts.14.up_proj", "model.layers.41.mlp.experts.15.up_proj", "model.layers.41.mlp.experts.16.up_proj", "model.layers.41.mlp.experts.17.up_proj", "model.layers.41.mlp.experts.18.up_proj", "model.layers.41.mlp.experts.19.up_proj", "model.layers.41.mlp.experts.20.up_proj", "model.layers.41.mlp.experts.21.up_proj", "model.layers.41.mlp.experts.22.up_proj", "model.layers.41.mlp.experts.23.up_proj", "model.layers.41.mlp.experts.24.up_proj", "model.layers.41.mlp.experts.25.up_proj", "model.layers.41.mlp.experts.26.up_proj", "model.layers.41.mlp.experts.27.up_proj", "model.layers.41.mlp.experts.28.up_proj", "model.layers.41.mlp.experts.29.up_proj", "model.layers.41.mlp.experts.30.up_proj", "model.layers.41.mlp.experts.31.up_proj", "model.layers.41.mlp.experts.32.up_proj", "model.layers.41.mlp.experts.33.up_proj", "model.layers.41.mlp.experts.34.up_proj", "model.layers.41.mlp.experts.35.up_proj", "model.layers.41.mlp.experts.36.up_proj", "model.layers.41.mlp.experts.37.up_proj", "model.layers.41.mlp.experts.38.up_proj", "model.layers.41.mlp.experts.39.up_proj", "model.layers.41.mlp.experts.40.up_proj", "model.layers.41.mlp.experts.41.up_proj", "model.layers.41.mlp.experts.42.up_proj", "model.layers.41.mlp.experts.43.up_proj", "model.layers.41.mlp.experts.44.up_proj", "model.layers.41.mlp.experts.45.up_proj", "model.layers.41.mlp.experts.46.up_proj", "model.layers.41.mlp.experts.47.up_proj", "model.layers.41.mlp.experts.48.up_proj", "model.layers.41.mlp.experts.49.up_proj", "model.layers.41.mlp.experts.50.up_proj", "model.layers.41.mlp.experts.51.up_proj", "model.layers.41.mlp.experts.52.up_proj", "model.layers.41.mlp.experts.53.up_proj", "model.layers.41.mlp.experts.54.up_proj", "model.layers.41.mlp.experts.55.up_proj", "model.layers.41.mlp.experts.56.up_proj", "model.layers.41.mlp.experts.57.up_proj", "model.layers.41.mlp.experts.58.up_proj", "model.layers.41.mlp.experts.59.up_proj", "model.layers.41.mlp.experts.60.up_proj", "model.layers.41.mlp.experts.61.up_proj", "model.layers.41.mlp.experts.62.up_proj", "model.layers.41.mlp.experts.63.up_proj", "model.layers.41.mlp.experts.64.up_proj", "model.layers.41.mlp.experts.65.up_proj", "model.layers.41.mlp.experts.66.up_proj", "model.layers.41.mlp.experts.67.up_proj", "model.layers.41.mlp.experts.68.up_proj", "model.layers.41.mlp.experts.69.up_proj", "model.layers.41.mlp.experts.70.up_proj", "model.layers.41.mlp.experts.71.up_proj", "model.layers.41.mlp.experts.72.up_proj", "model.layers.41.mlp.experts.73.up_proj", "model.layers.41.mlp.experts.74.up_proj", "model.layers.41.mlp.experts.75.up_proj", "model.layers.41.mlp.experts.76.up_proj", "model.layers.41.mlp.experts.77.up_proj", "model.layers.41.mlp.experts.78.up_proj", "model.layers.41.mlp.experts.79.up_proj", "model.layers.41.mlp.experts.80.up_proj", "model.layers.41.mlp.experts.81.up_proj", "model.layers.41.mlp.experts.82.up_proj", "model.layers.41.mlp.experts.83.up_proj", "model.layers.41.mlp.experts.84.up_proj", "model.layers.41.mlp.experts.85.up_proj", "model.layers.41.mlp.experts.86.up_proj", "model.layers.41.mlp.experts.87.up_proj", "model.layers.41.mlp.experts.88.up_proj", "model.layers.41.mlp.experts.89.up_proj", "model.layers.41.mlp.experts.90.up_proj", "model.layers.41.mlp.experts.91.up_proj", "model.layers.41.mlp.experts.92.up_proj", "model.layers.41.mlp.experts.93.up_proj", "model.layers.41.mlp.experts.94.up_proj", "model.layers.41.mlp.experts.95.up_proj", "model.layers.41.mlp.experts.96.up_proj", "model.layers.41.mlp.experts.97.up_proj", "model.layers.41.mlp.experts.98.up_proj", "model.layers.41.mlp.experts.99.up_proj", "model.layers.41.mlp.experts.100.up_proj", "model.layers.41.mlp.experts.101.up_proj", "model.layers.41.mlp.experts.102.up_proj", "model.layers.41.mlp.experts.103.up_proj", "model.layers.41.mlp.experts.104.up_proj", "model.layers.41.mlp.experts.105.up_proj", "model.layers.41.mlp.experts.106.up_proj", "model.layers.41.mlp.experts.107.up_proj", "model.layers.41.mlp.experts.108.up_proj", "model.layers.41.mlp.experts.109.up_proj", "model.layers.41.mlp.experts.110.up_proj", "model.layers.41.mlp.experts.111.up_proj", "model.layers.41.mlp.experts.112.up_proj", "model.layers.41.mlp.experts.113.up_proj", "model.layers.41.mlp.experts.114.up_proj", "model.layers.41.mlp.experts.115.up_proj", "model.layers.41.mlp.experts.116.up_proj", "model.layers.41.mlp.experts.117.up_proj", "model.layers.41.mlp.experts.118.up_proj", "model.layers.41.mlp.experts.119.up_proj", "model.layers.41.mlp.experts.120.up_proj", "model.layers.41.mlp.experts.121.up_proj", "model.layers.41.mlp.experts.122.up_proj", "model.layers.41.mlp.experts.123.up_proj", "model.layers.41.mlp.experts.124.up_proj", "model.layers.41.mlp.experts.125.up_proj", "model.layers.41.mlp.experts.126.up_proj", "model.layers.41.mlp.experts.127.up_proj", "model.layers.41.mlp.experts.128.up_proj", "model.layers.41.mlp.experts.129.up_proj", "model.layers.41.mlp.experts.130.up_proj", "model.layers.41.mlp.experts.131.up_proj", "model.layers.41.mlp.experts.132.up_proj", "model.layers.41.mlp.experts.133.up_proj", "model.layers.41.mlp.experts.134.up_proj", "model.layers.41.mlp.experts.135.up_proj", "model.layers.41.mlp.experts.136.up_proj", "model.layers.41.mlp.experts.137.up_proj", "model.layers.41.mlp.experts.138.up_proj", "model.layers.41.mlp.experts.139.up_proj", "model.layers.41.mlp.experts.140.up_proj", "model.layers.41.mlp.experts.141.up_proj", "model.layers.41.mlp.experts.142.up_proj", "model.layers.41.mlp.experts.143.up_proj", "model.layers.41.mlp.experts.144.up_proj", "model.layers.41.mlp.experts.145.up_proj", "model.layers.41.mlp.experts.146.up_proj", "model.layers.41.mlp.experts.147.up_proj", "model.layers.41.mlp.experts.148.up_proj", "model.layers.41.mlp.experts.149.up_proj", "model.layers.41.mlp.experts.150.up_proj", "model.layers.41.mlp.experts.151.up_proj", "model.layers.41.mlp.experts.152.up_proj", "model.layers.41.mlp.experts.153.up_proj", "model.layers.41.mlp.experts.154.up_proj", "model.layers.41.mlp.experts.155.up_proj", "model.layers.41.mlp.experts.156.up_proj", "model.layers.41.mlp.experts.157.up_proj", "model.layers.41.mlp.experts.158.up_proj", "model.layers.41.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.5062600141390941e-06, "dbits": 5033164800 } ] }, { "idx": 248, "layers": [ "model.layers.41.mlp.experts.0.down_proj", "model.layers.41.mlp.experts.1.down_proj", "model.layers.41.mlp.experts.2.down_proj", "model.layers.41.mlp.experts.3.down_proj", "model.layers.41.mlp.experts.4.down_proj", "model.layers.41.mlp.experts.5.down_proj", "model.layers.41.mlp.experts.6.down_proj", "model.layers.41.mlp.experts.7.down_proj", "model.layers.41.mlp.experts.8.down_proj", "model.layers.41.mlp.experts.9.down_proj", "model.layers.41.mlp.experts.10.down_proj", "model.layers.41.mlp.experts.11.down_proj", "model.layers.41.mlp.experts.12.down_proj", "model.layers.41.mlp.experts.13.down_proj", "model.layers.41.mlp.experts.14.down_proj", "model.layers.41.mlp.experts.15.down_proj", "model.layers.41.mlp.experts.16.down_proj", "model.layers.41.mlp.experts.17.down_proj", "model.layers.41.mlp.experts.18.down_proj", "model.layers.41.mlp.experts.19.down_proj", "model.layers.41.mlp.experts.20.down_proj", "model.layers.41.mlp.experts.21.down_proj", "model.layers.41.mlp.experts.22.down_proj", "model.layers.41.mlp.experts.23.down_proj", "model.layers.41.mlp.experts.24.down_proj", "model.layers.41.mlp.experts.25.down_proj", "model.layers.41.mlp.experts.26.down_proj", "model.layers.41.mlp.experts.27.down_proj", "model.layers.41.mlp.experts.28.down_proj", "model.layers.41.mlp.experts.29.down_proj", "model.layers.41.mlp.experts.30.down_proj", "model.layers.41.mlp.experts.31.down_proj", "model.layers.41.mlp.experts.32.down_proj", "model.layers.41.mlp.experts.33.down_proj", "model.layers.41.mlp.experts.34.down_proj", "model.layers.41.mlp.experts.35.down_proj", "model.layers.41.mlp.experts.36.down_proj", "model.layers.41.mlp.experts.37.down_proj", "model.layers.41.mlp.experts.38.down_proj", "model.layers.41.mlp.experts.39.down_proj", "model.layers.41.mlp.experts.40.down_proj", "model.layers.41.mlp.experts.41.down_proj", "model.layers.41.mlp.experts.42.down_proj", "model.layers.41.mlp.experts.43.down_proj", "model.layers.41.mlp.experts.44.down_proj", "model.layers.41.mlp.experts.45.down_proj", "model.layers.41.mlp.experts.46.down_proj", "model.layers.41.mlp.experts.47.down_proj", "model.layers.41.mlp.experts.48.down_proj", "model.layers.41.mlp.experts.49.down_proj", "model.layers.41.mlp.experts.50.down_proj", "model.layers.41.mlp.experts.51.down_proj", "model.layers.41.mlp.experts.52.down_proj", "model.layers.41.mlp.experts.53.down_proj", "model.layers.41.mlp.experts.54.down_proj", "model.layers.41.mlp.experts.55.down_proj", "model.layers.41.mlp.experts.56.down_proj", "model.layers.41.mlp.experts.57.down_proj", "model.layers.41.mlp.experts.58.down_proj", "model.layers.41.mlp.experts.59.down_proj", "model.layers.41.mlp.experts.60.down_proj", "model.layers.41.mlp.experts.61.down_proj", "model.layers.41.mlp.experts.62.down_proj", "model.layers.41.mlp.experts.63.down_proj", "model.layers.41.mlp.experts.64.down_proj", "model.layers.41.mlp.experts.65.down_proj", "model.layers.41.mlp.experts.66.down_proj", "model.layers.41.mlp.experts.67.down_proj", "model.layers.41.mlp.experts.68.down_proj", "model.layers.41.mlp.experts.69.down_proj", "model.layers.41.mlp.experts.70.down_proj", "model.layers.41.mlp.experts.71.down_proj", "model.layers.41.mlp.experts.72.down_proj", "model.layers.41.mlp.experts.73.down_proj", "model.layers.41.mlp.experts.74.down_proj", "model.layers.41.mlp.experts.75.down_proj", "model.layers.41.mlp.experts.76.down_proj", "model.layers.41.mlp.experts.77.down_proj", "model.layers.41.mlp.experts.78.down_proj", "model.layers.41.mlp.experts.79.down_proj", "model.layers.41.mlp.experts.80.down_proj", "model.layers.41.mlp.experts.81.down_proj", "model.layers.41.mlp.experts.82.down_proj", "model.layers.41.mlp.experts.83.down_proj", "model.layers.41.mlp.experts.84.down_proj", "model.layers.41.mlp.experts.85.down_proj", "model.layers.41.mlp.experts.86.down_proj", "model.layers.41.mlp.experts.87.down_proj", "model.layers.41.mlp.experts.88.down_proj", "model.layers.41.mlp.experts.89.down_proj", "model.layers.41.mlp.experts.90.down_proj", "model.layers.41.mlp.experts.91.down_proj", "model.layers.41.mlp.experts.92.down_proj", "model.layers.41.mlp.experts.93.down_proj", "model.layers.41.mlp.experts.94.down_proj", "model.layers.41.mlp.experts.95.down_proj", "model.layers.41.mlp.experts.96.down_proj", "model.layers.41.mlp.experts.97.down_proj", "model.layers.41.mlp.experts.98.down_proj", "model.layers.41.mlp.experts.99.down_proj", "model.layers.41.mlp.experts.100.down_proj", "model.layers.41.mlp.experts.101.down_proj", "model.layers.41.mlp.experts.102.down_proj", "model.layers.41.mlp.experts.103.down_proj", "model.layers.41.mlp.experts.104.down_proj", "model.layers.41.mlp.experts.105.down_proj", "model.layers.41.mlp.experts.106.down_proj", "model.layers.41.mlp.experts.107.down_proj", "model.layers.41.mlp.experts.108.down_proj", "model.layers.41.mlp.experts.109.down_proj", "model.layers.41.mlp.experts.110.down_proj", "model.layers.41.mlp.experts.111.down_proj", "model.layers.41.mlp.experts.112.down_proj", "model.layers.41.mlp.experts.113.down_proj", "model.layers.41.mlp.experts.114.down_proj", "model.layers.41.mlp.experts.115.down_proj", "model.layers.41.mlp.experts.116.down_proj", "model.layers.41.mlp.experts.117.down_proj", "model.layers.41.mlp.experts.118.down_proj", "model.layers.41.mlp.experts.119.down_proj", "model.layers.41.mlp.experts.120.down_proj", "model.layers.41.mlp.experts.121.down_proj", "model.layers.41.mlp.experts.122.down_proj", "model.layers.41.mlp.experts.123.down_proj", "model.layers.41.mlp.experts.124.down_proj", "model.layers.41.mlp.experts.125.down_proj", "model.layers.41.mlp.experts.126.down_proj", "model.layers.41.mlp.experts.127.down_proj", "model.layers.41.mlp.experts.128.down_proj", "model.layers.41.mlp.experts.129.down_proj", "model.layers.41.mlp.experts.130.down_proj", "model.layers.41.mlp.experts.131.down_proj", "model.layers.41.mlp.experts.132.down_proj", "model.layers.41.mlp.experts.133.down_proj", "model.layers.41.mlp.experts.134.down_proj", "model.layers.41.mlp.experts.135.down_proj", "model.layers.41.mlp.experts.136.down_proj", "model.layers.41.mlp.experts.137.down_proj", "model.layers.41.mlp.experts.138.down_proj", "model.layers.41.mlp.experts.139.down_proj", "model.layers.41.mlp.experts.140.down_proj", "model.layers.41.mlp.experts.141.down_proj", "model.layers.41.mlp.experts.142.down_proj", "model.layers.41.mlp.experts.143.down_proj", "model.layers.41.mlp.experts.144.down_proj", "model.layers.41.mlp.experts.145.down_proj", "model.layers.41.mlp.experts.146.down_proj", "model.layers.41.mlp.experts.147.down_proj", "model.layers.41.mlp.experts.148.down_proj", "model.layers.41.mlp.experts.149.down_proj", "model.layers.41.mlp.experts.150.down_proj", "model.layers.41.mlp.experts.151.down_proj", "model.layers.41.mlp.experts.152.down_proj", "model.layers.41.mlp.experts.153.down_proj", "model.layers.41.mlp.experts.154.down_proj", "model.layers.41.mlp.experts.155.down_proj", "model.layers.41.mlp.experts.156.down_proj", "model.layers.41.mlp.experts.157.down_proj", "model.layers.41.mlp.experts.158.down_proj", "model.layers.41.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.6049860278144045e-06, "dbits": 2516582400 } ] }, { "idx": 249, "layers": [ "model.layers.42.self_attn.q_proj" ], "candidates": [ { "dkld": 6.575798033736733e-06, "dbits": 125829120 } ] }, { "idx": 250, "layers": [ "model.layers.42.self_attn.k_proj", "model.layers.42.self_attn.v_proj" ], "candidates": [ { "dkld": -4.1848514229058786e-06, "dbits": 20971520 } ] }, { "idx": 251, "layers": [ "model.layers.42.self_attn.o_proj" ], "candidates": [ { "dkld": 2.9459421057254293e-06, "dbits": 125829120 } ] }, { "idx": 252, "layers": [ "model.layers.42.mlp.shared_experts.gate_proj", "model.layers.42.mlp.shared_experts.up_proj", "model.layers.42.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.9896368030458255e-06, "dbits": 47185920 } ] }, { "idx": 253, "layers": [ "model.layers.42.mlp.experts.0.gate_proj", "model.layers.42.mlp.experts.1.gate_proj", "model.layers.42.mlp.experts.2.gate_proj", "model.layers.42.mlp.experts.3.gate_proj", "model.layers.42.mlp.experts.4.gate_proj", "model.layers.42.mlp.experts.5.gate_proj", "model.layers.42.mlp.experts.6.gate_proj", "model.layers.42.mlp.experts.7.gate_proj", "model.layers.42.mlp.experts.8.gate_proj", "model.layers.42.mlp.experts.9.gate_proj", "model.layers.42.mlp.experts.10.gate_proj", "model.layers.42.mlp.experts.11.gate_proj", "model.layers.42.mlp.experts.12.gate_proj", "model.layers.42.mlp.experts.13.gate_proj", "model.layers.42.mlp.experts.14.gate_proj", "model.layers.42.mlp.experts.15.gate_proj", "model.layers.42.mlp.experts.16.gate_proj", "model.layers.42.mlp.experts.17.gate_proj", "model.layers.42.mlp.experts.18.gate_proj", "model.layers.42.mlp.experts.19.gate_proj", "model.layers.42.mlp.experts.20.gate_proj", "model.layers.42.mlp.experts.21.gate_proj", "model.layers.42.mlp.experts.22.gate_proj", "model.layers.42.mlp.experts.23.gate_proj", "model.layers.42.mlp.experts.24.gate_proj", "model.layers.42.mlp.experts.25.gate_proj", "model.layers.42.mlp.experts.26.gate_proj", "model.layers.42.mlp.experts.27.gate_proj", "model.layers.42.mlp.experts.28.gate_proj", "model.layers.42.mlp.experts.29.gate_proj", "model.layers.42.mlp.experts.30.gate_proj", "model.layers.42.mlp.experts.31.gate_proj", "model.layers.42.mlp.experts.32.gate_proj", "model.layers.42.mlp.experts.33.gate_proj", "model.layers.42.mlp.experts.34.gate_proj", "model.layers.42.mlp.experts.35.gate_proj", "model.layers.42.mlp.experts.36.gate_proj", "model.layers.42.mlp.experts.37.gate_proj", "model.layers.42.mlp.experts.38.gate_proj", "model.layers.42.mlp.experts.39.gate_proj", "model.layers.42.mlp.experts.40.gate_proj", "model.layers.42.mlp.experts.41.gate_proj", "model.layers.42.mlp.experts.42.gate_proj", "model.layers.42.mlp.experts.43.gate_proj", "model.layers.42.mlp.experts.44.gate_proj", "model.layers.42.mlp.experts.45.gate_proj", "model.layers.42.mlp.experts.46.gate_proj", "model.layers.42.mlp.experts.47.gate_proj", "model.layers.42.mlp.experts.48.gate_proj", "model.layers.42.mlp.experts.49.gate_proj", "model.layers.42.mlp.experts.50.gate_proj", "model.layers.42.mlp.experts.51.gate_proj", "model.layers.42.mlp.experts.52.gate_proj", "model.layers.42.mlp.experts.53.gate_proj", "model.layers.42.mlp.experts.54.gate_proj", "model.layers.42.mlp.experts.55.gate_proj", "model.layers.42.mlp.experts.56.gate_proj", "model.layers.42.mlp.experts.57.gate_proj", "model.layers.42.mlp.experts.58.gate_proj", "model.layers.42.mlp.experts.59.gate_proj", "model.layers.42.mlp.experts.60.gate_proj", "model.layers.42.mlp.experts.61.gate_proj", "model.layers.42.mlp.experts.62.gate_proj", "model.layers.42.mlp.experts.63.gate_proj", "model.layers.42.mlp.experts.64.gate_proj", "model.layers.42.mlp.experts.65.gate_proj", "model.layers.42.mlp.experts.66.gate_proj", "model.layers.42.mlp.experts.67.gate_proj", "model.layers.42.mlp.experts.68.gate_proj", "model.layers.42.mlp.experts.69.gate_proj", "model.layers.42.mlp.experts.70.gate_proj", "model.layers.42.mlp.experts.71.gate_proj", "model.layers.42.mlp.experts.72.gate_proj", "model.layers.42.mlp.experts.73.gate_proj", "model.layers.42.mlp.experts.74.gate_proj", "model.layers.42.mlp.experts.75.gate_proj", "model.layers.42.mlp.experts.76.gate_proj", "model.layers.42.mlp.experts.77.gate_proj", "model.layers.42.mlp.experts.78.gate_proj", "model.layers.42.mlp.experts.79.gate_proj", "model.layers.42.mlp.experts.80.gate_proj", "model.layers.42.mlp.experts.81.gate_proj", "model.layers.42.mlp.experts.82.gate_proj", "model.layers.42.mlp.experts.83.gate_proj", "model.layers.42.mlp.experts.84.gate_proj", "model.layers.42.mlp.experts.85.gate_proj", "model.layers.42.mlp.experts.86.gate_proj", "model.layers.42.mlp.experts.87.gate_proj", "model.layers.42.mlp.experts.88.gate_proj", "model.layers.42.mlp.experts.89.gate_proj", "model.layers.42.mlp.experts.90.gate_proj", "model.layers.42.mlp.experts.91.gate_proj", "model.layers.42.mlp.experts.92.gate_proj", "model.layers.42.mlp.experts.93.gate_proj", "model.layers.42.mlp.experts.94.gate_proj", "model.layers.42.mlp.experts.95.gate_proj", "model.layers.42.mlp.experts.96.gate_proj", "model.layers.42.mlp.experts.97.gate_proj", "model.layers.42.mlp.experts.98.gate_proj", "model.layers.42.mlp.experts.99.gate_proj", "model.layers.42.mlp.experts.100.gate_proj", "model.layers.42.mlp.experts.101.gate_proj", "model.layers.42.mlp.experts.102.gate_proj", "model.layers.42.mlp.experts.103.gate_proj", "model.layers.42.mlp.experts.104.gate_proj", "model.layers.42.mlp.experts.105.gate_proj", "model.layers.42.mlp.experts.106.gate_proj", "model.layers.42.mlp.experts.107.gate_proj", "model.layers.42.mlp.experts.108.gate_proj", "model.layers.42.mlp.experts.109.gate_proj", "model.layers.42.mlp.experts.110.gate_proj", "model.layers.42.mlp.experts.111.gate_proj", "model.layers.42.mlp.experts.112.gate_proj", "model.layers.42.mlp.experts.113.gate_proj", "model.layers.42.mlp.experts.114.gate_proj", "model.layers.42.mlp.experts.115.gate_proj", "model.layers.42.mlp.experts.116.gate_proj", "model.layers.42.mlp.experts.117.gate_proj", "model.layers.42.mlp.experts.118.gate_proj", "model.layers.42.mlp.experts.119.gate_proj", "model.layers.42.mlp.experts.120.gate_proj", "model.layers.42.mlp.experts.121.gate_proj", "model.layers.42.mlp.experts.122.gate_proj", "model.layers.42.mlp.experts.123.gate_proj", "model.layers.42.mlp.experts.124.gate_proj", "model.layers.42.mlp.experts.125.gate_proj", "model.layers.42.mlp.experts.126.gate_proj", "model.layers.42.mlp.experts.127.gate_proj", "model.layers.42.mlp.experts.128.gate_proj", "model.layers.42.mlp.experts.129.gate_proj", "model.layers.42.mlp.experts.130.gate_proj", "model.layers.42.mlp.experts.131.gate_proj", "model.layers.42.mlp.experts.132.gate_proj", "model.layers.42.mlp.experts.133.gate_proj", "model.layers.42.mlp.experts.134.gate_proj", "model.layers.42.mlp.experts.135.gate_proj", "model.layers.42.mlp.experts.136.gate_proj", "model.layers.42.mlp.experts.137.gate_proj", "model.layers.42.mlp.experts.138.gate_proj", "model.layers.42.mlp.experts.139.gate_proj", "model.layers.42.mlp.experts.140.gate_proj", "model.layers.42.mlp.experts.141.gate_proj", "model.layers.42.mlp.experts.142.gate_proj", "model.layers.42.mlp.experts.143.gate_proj", "model.layers.42.mlp.experts.144.gate_proj", "model.layers.42.mlp.experts.145.gate_proj", "model.layers.42.mlp.experts.146.gate_proj", "model.layers.42.mlp.experts.147.gate_proj", "model.layers.42.mlp.experts.148.gate_proj", "model.layers.42.mlp.experts.149.gate_proj", "model.layers.42.mlp.experts.150.gate_proj", "model.layers.42.mlp.experts.151.gate_proj", "model.layers.42.mlp.experts.152.gate_proj", "model.layers.42.mlp.experts.153.gate_proj", "model.layers.42.mlp.experts.154.gate_proj", "model.layers.42.mlp.experts.155.gate_proj", "model.layers.42.mlp.experts.156.gate_proj", "model.layers.42.mlp.experts.157.gate_proj", "model.layers.42.mlp.experts.158.gate_proj", "model.layers.42.mlp.experts.159.gate_proj", "model.layers.42.mlp.experts.0.up_proj", "model.layers.42.mlp.experts.1.up_proj", "model.layers.42.mlp.experts.2.up_proj", "model.layers.42.mlp.experts.3.up_proj", "model.layers.42.mlp.experts.4.up_proj", "model.layers.42.mlp.experts.5.up_proj", "model.layers.42.mlp.experts.6.up_proj", "model.layers.42.mlp.experts.7.up_proj", "model.layers.42.mlp.experts.8.up_proj", "model.layers.42.mlp.experts.9.up_proj", "model.layers.42.mlp.experts.10.up_proj", "model.layers.42.mlp.experts.11.up_proj", "model.layers.42.mlp.experts.12.up_proj", "model.layers.42.mlp.experts.13.up_proj", "model.layers.42.mlp.experts.14.up_proj", "model.layers.42.mlp.experts.15.up_proj", "model.layers.42.mlp.experts.16.up_proj", "model.layers.42.mlp.experts.17.up_proj", "model.layers.42.mlp.experts.18.up_proj", "model.layers.42.mlp.experts.19.up_proj", "model.layers.42.mlp.experts.20.up_proj", "model.layers.42.mlp.experts.21.up_proj", "model.layers.42.mlp.experts.22.up_proj", "model.layers.42.mlp.experts.23.up_proj", "model.layers.42.mlp.experts.24.up_proj", "model.layers.42.mlp.experts.25.up_proj", "model.layers.42.mlp.experts.26.up_proj", "model.layers.42.mlp.experts.27.up_proj", "model.layers.42.mlp.experts.28.up_proj", "model.layers.42.mlp.experts.29.up_proj", "model.layers.42.mlp.experts.30.up_proj", "model.layers.42.mlp.experts.31.up_proj", "model.layers.42.mlp.experts.32.up_proj", "model.layers.42.mlp.experts.33.up_proj", "model.layers.42.mlp.experts.34.up_proj", "model.layers.42.mlp.experts.35.up_proj", "model.layers.42.mlp.experts.36.up_proj", "model.layers.42.mlp.experts.37.up_proj", "model.layers.42.mlp.experts.38.up_proj", "model.layers.42.mlp.experts.39.up_proj", "model.layers.42.mlp.experts.40.up_proj", "model.layers.42.mlp.experts.41.up_proj", "model.layers.42.mlp.experts.42.up_proj", "model.layers.42.mlp.experts.43.up_proj", "model.layers.42.mlp.experts.44.up_proj", "model.layers.42.mlp.experts.45.up_proj", "model.layers.42.mlp.experts.46.up_proj", "model.layers.42.mlp.experts.47.up_proj", "model.layers.42.mlp.experts.48.up_proj", "model.layers.42.mlp.experts.49.up_proj", "model.layers.42.mlp.experts.50.up_proj", "model.layers.42.mlp.experts.51.up_proj", "model.layers.42.mlp.experts.52.up_proj", "model.layers.42.mlp.experts.53.up_proj", "model.layers.42.mlp.experts.54.up_proj", "model.layers.42.mlp.experts.55.up_proj", "model.layers.42.mlp.experts.56.up_proj", "model.layers.42.mlp.experts.57.up_proj", "model.layers.42.mlp.experts.58.up_proj", "model.layers.42.mlp.experts.59.up_proj", "model.layers.42.mlp.experts.60.up_proj", "model.layers.42.mlp.experts.61.up_proj", "model.layers.42.mlp.experts.62.up_proj", "model.layers.42.mlp.experts.63.up_proj", "model.layers.42.mlp.experts.64.up_proj", "model.layers.42.mlp.experts.65.up_proj", "model.layers.42.mlp.experts.66.up_proj", "model.layers.42.mlp.experts.67.up_proj", "model.layers.42.mlp.experts.68.up_proj", "model.layers.42.mlp.experts.69.up_proj", "model.layers.42.mlp.experts.70.up_proj", "model.layers.42.mlp.experts.71.up_proj", "model.layers.42.mlp.experts.72.up_proj", "model.layers.42.mlp.experts.73.up_proj", "model.layers.42.mlp.experts.74.up_proj", "model.layers.42.mlp.experts.75.up_proj", "model.layers.42.mlp.experts.76.up_proj", "model.layers.42.mlp.experts.77.up_proj", "model.layers.42.mlp.experts.78.up_proj", "model.layers.42.mlp.experts.79.up_proj", "model.layers.42.mlp.experts.80.up_proj", "model.layers.42.mlp.experts.81.up_proj", "model.layers.42.mlp.experts.82.up_proj", "model.layers.42.mlp.experts.83.up_proj", "model.layers.42.mlp.experts.84.up_proj", "model.layers.42.mlp.experts.85.up_proj", "model.layers.42.mlp.experts.86.up_proj", "model.layers.42.mlp.experts.87.up_proj", "model.layers.42.mlp.experts.88.up_proj", "model.layers.42.mlp.experts.89.up_proj", "model.layers.42.mlp.experts.90.up_proj", "model.layers.42.mlp.experts.91.up_proj", "model.layers.42.mlp.experts.92.up_proj", "model.layers.42.mlp.experts.93.up_proj", "model.layers.42.mlp.experts.94.up_proj", "model.layers.42.mlp.experts.95.up_proj", "model.layers.42.mlp.experts.96.up_proj", "model.layers.42.mlp.experts.97.up_proj", "model.layers.42.mlp.experts.98.up_proj", "model.layers.42.mlp.experts.99.up_proj", "model.layers.42.mlp.experts.100.up_proj", "model.layers.42.mlp.experts.101.up_proj", "model.layers.42.mlp.experts.102.up_proj", "model.layers.42.mlp.experts.103.up_proj", "model.layers.42.mlp.experts.104.up_proj", "model.layers.42.mlp.experts.105.up_proj", "model.layers.42.mlp.experts.106.up_proj", "model.layers.42.mlp.experts.107.up_proj", "model.layers.42.mlp.experts.108.up_proj", "model.layers.42.mlp.experts.109.up_proj", "model.layers.42.mlp.experts.110.up_proj", "model.layers.42.mlp.experts.111.up_proj", "model.layers.42.mlp.experts.112.up_proj", "model.layers.42.mlp.experts.113.up_proj", "model.layers.42.mlp.experts.114.up_proj", "model.layers.42.mlp.experts.115.up_proj", "model.layers.42.mlp.experts.116.up_proj", "model.layers.42.mlp.experts.117.up_proj", "model.layers.42.mlp.experts.118.up_proj", "model.layers.42.mlp.experts.119.up_proj", "model.layers.42.mlp.experts.120.up_proj", "model.layers.42.mlp.experts.121.up_proj", "model.layers.42.mlp.experts.122.up_proj", "model.layers.42.mlp.experts.123.up_proj", "model.layers.42.mlp.experts.124.up_proj", "model.layers.42.mlp.experts.125.up_proj", "model.layers.42.mlp.experts.126.up_proj", "model.layers.42.mlp.experts.127.up_proj", "model.layers.42.mlp.experts.128.up_proj", "model.layers.42.mlp.experts.129.up_proj", "model.layers.42.mlp.experts.130.up_proj", "model.layers.42.mlp.experts.131.up_proj", "model.layers.42.mlp.experts.132.up_proj", "model.layers.42.mlp.experts.133.up_proj", "model.layers.42.mlp.experts.134.up_proj", "model.layers.42.mlp.experts.135.up_proj", "model.layers.42.mlp.experts.136.up_proj", "model.layers.42.mlp.experts.137.up_proj", "model.layers.42.mlp.experts.138.up_proj", "model.layers.42.mlp.experts.139.up_proj", "model.layers.42.mlp.experts.140.up_proj", "model.layers.42.mlp.experts.141.up_proj", "model.layers.42.mlp.experts.142.up_proj", "model.layers.42.mlp.experts.143.up_proj", "model.layers.42.mlp.experts.144.up_proj", "model.layers.42.mlp.experts.145.up_proj", "model.layers.42.mlp.experts.146.up_proj", "model.layers.42.mlp.experts.147.up_proj", "model.layers.42.mlp.experts.148.up_proj", "model.layers.42.mlp.experts.149.up_proj", "model.layers.42.mlp.experts.150.up_proj", "model.layers.42.mlp.experts.151.up_proj", "model.layers.42.mlp.experts.152.up_proj", "model.layers.42.mlp.experts.153.up_proj", "model.layers.42.mlp.experts.154.up_proj", "model.layers.42.mlp.experts.155.up_proj", "model.layers.42.mlp.experts.156.up_proj", "model.layers.42.mlp.experts.157.up_proj", "model.layers.42.mlp.experts.158.up_proj", "model.layers.42.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.4250690583139658e-07, "dbits": 5033164800 } ] }, { "idx": 254, "layers": [ "model.layers.42.mlp.experts.0.down_proj", "model.layers.42.mlp.experts.1.down_proj", "model.layers.42.mlp.experts.2.down_proj", "model.layers.42.mlp.experts.3.down_proj", "model.layers.42.mlp.experts.4.down_proj", "model.layers.42.mlp.experts.5.down_proj", "model.layers.42.mlp.experts.6.down_proj", "model.layers.42.mlp.experts.7.down_proj", "model.layers.42.mlp.experts.8.down_proj", "model.layers.42.mlp.experts.9.down_proj", "model.layers.42.mlp.experts.10.down_proj", "model.layers.42.mlp.experts.11.down_proj", "model.layers.42.mlp.experts.12.down_proj", "model.layers.42.mlp.experts.13.down_proj", "model.layers.42.mlp.experts.14.down_proj", "model.layers.42.mlp.experts.15.down_proj", "model.layers.42.mlp.experts.16.down_proj", "model.layers.42.mlp.experts.17.down_proj", "model.layers.42.mlp.experts.18.down_proj", "model.layers.42.mlp.experts.19.down_proj", "model.layers.42.mlp.experts.20.down_proj", "model.layers.42.mlp.experts.21.down_proj", "model.layers.42.mlp.experts.22.down_proj", "model.layers.42.mlp.experts.23.down_proj", "model.layers.42.mlp.experts.24.down_proj", "model.layers.42.mlp.experts.25.down_proj", "model.layers.42.mlp.experts.26.down_proj", "model.layers.42.mlp.experts.27.down_proj", "model.layers.42.mlp.experts.28.down_proj", "model.layers.42.mlp.experts.29.down_proj", "model.layers.42.mlp.experts.30.down_proj", "model.layers.42.mlp.experts.31.down_proj", "model.layers.42.mlp.experts.32.down_proj", "model.layers.42.mlp.experts.33.down_proj", "model.layers.42.mlp.experts.34.down_proj", "model.layers.42.mlp.experts.35.down_proj", "model.layers.42.mlp.experts.36.down_proj", "model.layers.42.mlp.experts.37.down_proj", "model.layers.42.mlp.experts.38.down_proj", "model.layers.42.mlp.experts.39.down_proj", "model.layers.42.mlp.experts.40.down_proj", "model.layers.42.mlp.experts.41.down_proj", "model.layers.42.mlp.experts.42.down_proj", "model.layers.42.mlp.experts.43.down_proj", "model.layers.42.mlp.experts.44.down_proj", "model.layers.42.mlp.experts.45.down_proj", "model.layers.42.mlp.experts.46.down_proj", "model.layers.42.mlp.experts.47.down_proj", "model.layers.42.mlp.experts.48.down_proj", "model.layers.42.mlp.experts.49.down_proj", "model.layers.42.mlp.experts.50.down_proj", "model.layers.42.mlp.experts.51.down_proj", "model.layers.42.mlp.experts.52.down_proj", "model.layers.42.mlp.experts.53.down_proj", "model.layers.42.mlp.experts.54.down_proj", "model.layers.42.mlp.experts.55.down_proj", "model.layers.42.mlp.experts.56.down_proj", "model.layers.42.mlp.experts.57.down_proj", "model.layers.42.mlp.experts.58.down_proj", "model.layers.42.mlp.experts.59.down_proj", "model.layers.42.mlp.experts.60.down_proj", "model.layers.42.mlp.experts.61.down_proj", "model.layers.42.mlp.experts.62.down_proj", "model.layers.42.mlp.experts.63.down_proj", "model.layers.42.mlp.experts.64.down_proj", "model.layers.42.mlp.experts.65.down_proj", "model.layers.42.mlp.experts.66.down_proj", "model.layers.42.mlp.experts.67.down_proj", "model.layers.42.mlp.experts.68.down_proj", "model.layers.42.mlp.experts.69.down_proj", "model.layers.42.mlp.experts.70.down_proj", "model.layers.42.mlp.experts.71.down_proj", "model.layers.42.mlp.experts.72.down_proj", "model.layers.42.mlp.experts.73.down_proj", "model.layers.42.mlp.experts.74.down_proj", "model.layers.42.mlp.experts.75.down_proj", "model.layers.42.mlp.experts.76.down_proj", "model.layers.42.mlp.experts.77.down_proj", "model.layers.42.mlp.experts.78.down_proj", "model.layers.42.mlp.experts.79.down_proj", "model.layers.42.mlp.experts.80.down_proj", "model.layers.42.mlp.experts.81.down_proj", "model.layers.42.mlp.experts.82.down_proj", "model.layers.42.mlp.experts.83.down_proj", "model.layers.42.mlp.experts.84.down_proj", "model.layers.42.mlp.experts.85.down_proj", "model.layers.42.mlp.experts.86.down_proj", "model.layers.42.mlp.experts.87.down_proj", "model.layers.42.mlp.experts.88.down_proj", "model.layers.42.mlp.experts.89.down_proj", "model.layers.42.mlp.experts.90.down_proj", "model.layers.42.mlp.experts.91.down_proj", "model.layers.42.mlp.experts.92.down_proj", "model.layers.42.mlp.experts.93.down_proj", "model.layers.42.mlp.experts.94.down_proj", "model.layers.42.mlp.experts.95.down_proj", "model.layers.42.mlp.experts.96.down_proj", "model.layers.42.mlp.experts.97.down_proj", "model.layers.42.mlp.experts.98.down_proj", "model.layers.42.mlp.experts.99.down_proj", "model.layers.42.mlp.experts.100.down_proj", "model.layers.42.mlp.experts.101.down_proj", "model.layers.42.mlp.experts.102.down_proj", "model.layers.42.mlp.experts.103.down_proj", "model.layers.42.mlp.experts.104.down_proj", "model.layers.42.mlp.experts.105.down_proj", "model.layers.42.mlp.experts.106.down_proj", "model.layers.42.mlp.experts.107.down_proj", "model.layers.42.mlp.experts.108.down_proj", "model.layers.42.mlp.experts.109.down_proj", "model.layers.42.mlp.experts.110.down_proj", "model.layers.42.mlp.experts.111.down_proj", "model.layers.42.mlp.experts.112.down_proj", "model.layers.42.mlp.experts.113.down_proj", "model.layers.42.mlp.experts.114.down_proj", "model.layers.42.mlp.experts.115.down_proj", "model.layers.42.mlp.experts.116.down_proj", "model.layers.42.mlp.experts.117.down_proj", "model.layers.42.mlp.experts.118.down_proj", "model.layers.42.mlp.experts.119.down_proj", "model.layers.42.mlp.experts.120.down_proj", "model.layers.42.mlp.experts.121.down_proj", "model.layers.42.mlp.experts.122.down_proj", "model.layers.42.mlp.experts.123.down_proj", "model.layers.42.mlp.experts.124.down_proj", "model.layers.42.mlp.experts.125.down_proj", "model.layers.42.mlp.experts.126.down_proj", "model.layers.42.mlp.experts.127.down_proj", "model.layers.42.mlp.experts.128.down_proj", "model.layers.42.mlp.experts.129.down_proj", "model.layers.42.mlp.experts.130.down_proj", "model.layers.42.mlp.experts.131.down_proj", "model.layers.42.mlp.experts.132.down_proj", "model.layers.42.mlp.experts.133.down_proj", "model.layers.42.mlp.experts.134.down_proj", "model.layers.42.mlp.experts.135.down_proj", "model.layers.42.mlp.experts.136.down_proj", "model.layers.42.mlp.experts.137.down_proj", "model.layers.42.mlp.experts.138.down_proj", "model.layers.42.mlp.experts.139.down_proj", "model.layers.42.mlp.experts.140.down_proj", "model.layers.42.mlp.experts.141.down_proj", "model.layers.42.mlp.experts.142.down_proj", "model.layers.42.mlp.experts.143.down_proj", "model.layers.42.mlp.experts.144.down_proj", "model.layers.42.mlp.experts.145.down_proj", "model.layers.42.mlp.experts.146.down_proj", "model.layers.42.mlp.experts.147.down_proj", "model.layers.42.mlp.experts.148.down_proj", "model.layers.42.mlp.experts.149.down_proj", "model.layers.42.mlp.experts.150.down_proj", "model.layers.42.mlp.experts.151.down_proj", "model.layers.42.mlp.experts.152.down_proj", "model.layers.42.mlp.experts.153.down_proj", "model.layers.42.mlp.experts.154.down_proj", "model.layers.42.mlp.experts.155.down_proj", "model.layers.42.mlp.experts.156.down_proj", "model.layers.42.mlp.experts.157.down_proj", "model.layers.42.mlp.experts.158.down_proj", "model.layers.42.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 8.324813097715334e-06, "dbits": 2516582400 } ] }, { "idx": 255, "layers": [ "model.layers.43.self_attn.q_proj" ], "candidates": [ { "dkld": 6.929386290721552e-06, "dbits": 125829120 } ] }, { "idx": 256, "layers": [ "model.layers.43.self_attn.k_proj", "model.layers.43.self_attn.v_proj" ], "candidates": [ { "dkld": -3.384106094017625e-06, "dbits": 20971520 } ] }, { "idx": 257, "layers": [ "model.layers.43.self_attn.o_proj" ], "candidates": [ { "dkld": -7.670634659007625e-07, "dbits": 125829120 } ] }, { "idx": 258, "layers": [ "model.layers.43.mlp.shared_experts.gate_proj", "model.layers.43.mlp.shared_experts.up_proj", "model.layers.43.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.3005566112697124e-06, "dbits": 47185920 } ] }, { "idx": 259, "layers": [ "model.layers.43.mlp.experts.0.gate_proj", "model.layers.43.mlp.experts.1.gate_proj", "model.layers.43.mlp.experts.2.gate_proj", "model.layers.43.mlp.experts.3.gate_proj", "model.layers.43.mlp.experts.4.gate_proj", "model.layers.43.mlp.experts.5.gate_proj", "model.layers.43.mlp.experts.6.gate_proj", "model.layers.43.mlp.experts.7.gate_proj", "model.layers.43.mlp.experts.8.gate_proj", "model.layers.43.mlp.experts.9.gate_proj", "model.layers.43.mlp.experts.10.gate_proj", "model.layers.43.mlp.experts.11.gate_proj", "model.layers.43.mlp.experts.12.gate_proj", "model.layers.43.mlp.experts.13.gate_proj", "model.layers.43.mlp.experts.14.gate_proj", "model.layers.43.mlp.experts.15.gate_proj", "model.layers.43.mlp.experts.16.gate_proj", "model.layers.43.mlp.experts.17.gate_proj", "model.layers.43.mlp.experts.18.gate_proj", "model.layers.43.mlp.experts.19.gate_proj", "model.layers.43.mlp.experts.20.gate_proj", "model.layers.43.mlp.experts.21.gate_proj", "model.layers.43.mlp.experts.22.gate_proj", "model.layers.43.mlp.experts.23.gate_proj", "model.layers.43.mlp.experts.24.gate_proj", "model.layers.43.mlp.experts.25.gate_proj", "model.layers.43.mlp.experts.26.gate_proj", "model.layers.43.mlp.experts.27.gate_proj", "model.layers.43.mlp.experts.28.gate_proj", "model.layers.43.mlp.experts.29.gate_proj", "model.layers.43.mlp.experts.30.gate_proj", "model.layers.43.mlp.experts.31.gate_proj", "model.layers.43.mlp.experts.32.gate_proj", "model.layers.43.mlp.experts.33.gate_proj", "model.layers.43.mlp.experts.34.gate_proj", "model.layers.43.mlp.experts.35.gate_proj", "model.layers.43.mlp.experts.36.gate_proj", "model.layers.43.mlp.experts.37.gate_proj", "model.layers.43.mlp.experts.38.gate_proj", "model.layers.43.mlp.experts.39.gate_proj", "model.layers.43.mlp.experts.40.gate_proj", "model.layers.43.mlp.experts.41.gate_proj", "model.layers.43.mlp.experts.42.gate_proj", "model.layers.43.mlp.experts.43.gate_proj", "model.layers.43.mlp.experts.44.gate_proj", "model.layers.43.mlp.experts.45.gate_proj", "model.layers.43.mlp.experts.46.gate_proj", "model.layers.43.mlp.experts.47.gate_proj", "model.layers.43.mlp.experts.48.gate_proj", "model.layers.43.mlp.experts.49.gate_proj", "model.layers.43.mlp.experts.50.gate_proj", "model.layers.43.mlp.experts.51.gate_proj", "model.layers.43.mlp.experts.52.gate_proj", "model.layers.43.mlp.experts.53.gate_proj", "model.layers.43.mlp.experts.54.gate_proj", "model.layers.43.mlp.experts.55.gate_proj", "model.layers.43.mlp.experts.56.gate_proj", "model.layers.43.mlp.experts.57.gate_proj", "model.layers.43.mlp.experts.58.gate_proj", "model.layers.43.mlp.experts.59.gate_proj", "model.layers.43.mlp.experts.60.gate_proj", "model.layers.43.mlp.experts.61.gate_proj", "model.layers.43.mlp.experts.62.gate_proj", "model.layers.43.mlp.experts.63.gate_proj", "model.layers.43.mlp.experts.64.gate_proj", "model.layers.43.mlp.experts.65.gate_proj", "model.layers.43.mlp.experts.66.gate_proj", "model.layers.43.mlp.experts.67.gate_proj", "model.layers.43.mlp.experts.68.gate_proj", "model.layers.43.mlp.experts.69.gate_proj", "model.layers.43.mlp.experts.70.gate_proj", "model.layers.43.mlp.experts.71.gate_proj", "model.layers.43.mlp.experts.72.gate_proj", "model.layers.43.mlp.experts.73.gate_proj", "model.layers.43.mlp.experts.74.gate_proj", "model.layers.43.mlp.experts.75.gate_proj", "model.layers.43.mlp.experts.76.gate_proj", "model.layers.43.mlp.experts.77.gate_proj", "model.layers.43.mlp.experts.78.gate_proj", "model.layers.43.mlp.experts.79.gate_proj", "model.layers.43.mlp.experts.80.gate_proj", "model.layers.43.mlp.experts.81.gate_proj", "model.layers.43.mlp.experts.82.gate_proj", "model.layers.43.mlp.experts.83.gate_proj", "model.layers.43.mlp.experts.84.gate_proj", "model.layers.43.mlp.experts.85.gate_proj", "model.layers.43.mlp.experts.86.gate_proj", "model.layers.43.mlp.experts.87.gate_proj", "model.layers.43.mlp.experts.88.gate_proj", "model.layers.43.mlp.experts.89.gate_proj", "model.layers.43.mlp.experts.90.gate_proj", "model.layers.43.mlp.experts.91.gate_proj", "model.layers.43.mlp.experts.92.gate_proj", "model.layers.43.mlp.experts.93.gate_proj", "model.layers.43.mlp.experts.94.gate_proj", "model.layers.43.mlp.experts.95.gate_proj", "model.layers.43.mlp.experts.96.gate_proj", "model.layers.43.mlp.experts.97.gate_proj", "model.layers.43.mlp.experts.98.gate_proj", "model.layers.43.mlp.experts.99.gate_proj", "model.layers.43.mlp.experts.100.gate_proj", "model.layers.43.mlp.experts.101.gate_proj", "model.layers.43.mlp.experts.102.gate_proj", "model.layers.43.mlp.experts.103.gate_proj", "model.layers.43.mlp.experts.104.gate_proj", "model.layers.43.mlp.experts.105.gate_proj", "model.layers.43.mlp.experts.106.gate_proj", "model.layers.43.mlp.experts.107.gate_proj", "model.layers.43.mlp.experts.108.gate_proj", "model.layers.43.mlp.experts.109.gate_proj", "model.layers.43.mlp.experts.110.gate_proj", "model.layers.43.mlp.experts.111.gate_proj", "model.layers.43.mlp.experts.112.gate_proj", "model.layers.43.mlp.experts.113.gate_proj", "model.layers.43.mlp.experts.114.gate_proj", "model.layers.43.mlp.experts.115.gate_proj", "model.layers.43.mlp.experts.116.gate_proj", "model.layers.43.mlp.experts.117.gate_proj", "model.layers.43.mlp.experts.118.gate_proj", "model.layers.43.mlp.experts.119.gate_proj", "model.layers.43.mlp.experts.120.gate_proj", "model.layers.43.mlp.experts.121.gate_proj", "model.layers.43.mlp.experts.122.gate_proj", "model.layers.43.mlp.experts.123.gate_proj", "model.layers.43.mlp.experts.124.gate_proj", "model.layers.43.mlp.experts.125.gate_proj", "model.layers.43.mlp.experts.126.gate_proj", "model.layers.43.mlp.experts.127.gate_proj", "model.layers.43.mlp.experts.128.gate_proj", "model.layers.43.mlp.experts.129.gate_proj", "model.layers.43.mlp.experts.130.gate_proj", "model.layers.43.mlp.experts.131.gate_proj", "model.layers.43.mlp.experts.132.gate_proj", "model.layers.43.mlp.experts.133.gate_proj", "model.layers.43.mlp.experts.134.gate_proj", "model.layers.43.mlp.experts.135.gate_proj", "model.layers.43.mlp.experts.136.gate_proj", "model.layers.43.mlp.experts.137.gate_proj", "model.layers.43.mlp.experts.138.gate_proj", "model.layers.43.mlp.experts.139.gate_proj", "model.layers.43.mlp.experts.140.gate_proj", "model.layers.43.mlp.experts.141.gate_proj", "model.layers.43.mlp.experts.142.gate_proj", "model.layers.43.mlp.experts.143.gate_proj", "model.layers.43.mlp.experts.144.gate_proj", "model.layers.43.mlp.experts.145.gate_proj", "model.layers.43.mlp.experts.146.gate_proj", "model.layers.43.mlp.experts.147.gate_proj", "model.layers.43.mlp.experts.148.gate_proj", "model.layers.43.mlp.experts.149.gate_proj", "model.layers.43.mlp.experts.150.gate_proj", "model.layers.43.mlp.experts.151.gate_proj", "model.layers.43.mlp.experts.152.gate_proj", "model.layers.43.mlp.experts.153.gate_proj", "model.layers.43.mlp.experts.154.gate_proj", "model.layers.43.mlp.experts.155.gate_proj", "model.layers.43.mlp.experts.156.gate_proj", "model.layers.43.mlp.experts.157.gate_proj", "model.layers.43.mlp.experts.158.gate_proj", "model.layers.43.mlp.experts.159.gate_proj", "model.layers.43.mlp.experts.0.up_proj", "model.layers.43.mlp.experts.1.up_proj", "model.layers.43.mlp.experts.2.up_proj", "model.layers.43.mlp.experts.3.up_proj", "model.layers.43.mlp.experts.4.up_proj", "model.layers.43.mlp.experts.5.up_proj", "model.layers.43.mlp.experts.6.up_proj", "model.layers.43.mlp.experts.7.up_proj", "model.layers.43.mlp.experts.8.up_proj", "model.layers.43.mlp.experts.9.up_proj", "model.layers.43.mlp.experts.10.up_proj", "model.layers.43.mlp.experts.11.up_proj", "model.layers.43.mlp.experts.12.up_proj", "model.layers.43.mlp.experts.13.up_proj", "model.layers.43.mlp.experts.14.up_proj", "model.layers.43.mlp.experts.15.up_proj", "model.layers.43.mlp.experts.16.up_proj", "model.layers.43.mlp.experts.17.up_proj", "model.layers.43.mlp.experts.18.up_proj", "model.layers.43.mlp.experts.19.up_proj", "model.layers.43.mlp.experts.20.up_proj", "model.layers.43.mlp.experts.21.up_proj", "model.layers.43.mlp.experts.22.up_proj", "model.layers.43.mlp.experts.23.up_proj", "model.layers.43.mlp.experts.24.up_proj", "model.layers.43.mlp.experts.25.up_proj", "model.layers.43.mlp.experts.26.up_proj", "model.layers.43.mlp.experts.27.up_proj", "model.layers.43.mlp.experts.28.up_proj", "model.layers.43.mlp.experts.29.up_proj", "model.layers.43.mlp.experts.30.up_proj", "model.layers.43.mlp.experts.31.up_proj", "model.layers.43.mlp.experts.32.up_proj", "model.layers.43.mlp.experts.33.up_proj", "model.layers.43.mlp.experts.34.up_proj", "model.layers.43.mlp.experts.35.up_proj", "model.layers.43.mlp.experts.36.up_proj", "model.layers.43.mlp.experts.37.up_proj", "model.layers.43.mlp.experts.38.up_proj", "model.layers.43.mlp.experts.39.up_proj", "model.layers.43.mlp.experts.40.up_proj", "model.layers.43.mlp.experts.41.up_proj", "model.layers.43.mlp.experts.42.up_proj", "model.layers.43.mlp.experts.43.up_proj", "model.layers.43.mlp.experts.44.up_proj", "model.layers.43.mlp.experts.45.up_proj", "model.layers.43.mlp.experts.46.up_proj", "model.layers.43.mlp.experts.47.up_proj", "model.layers.43.mlp.experts.48.up_proj", "model.layers.43.mlp.experts.49.up_proj", "model.layers.43.mlp.experts.50.up_proj", "model.layers.43.mlp.experts.51.up_proj", "model.layers.43.mlp.experts.52.up_proj", "model.layers.43.mlp.experts.53.up_proj", "model.layers.43.mlp.experts.54.up_proj", "model.layers.43.mlp.experts.55.up_proj", "model.layers.43.mlp.experts.56.up_proj", "model.layers.43.mlp.experts.57.up_proj", "model.layers.43.mlp.experts.58.up_proj", "model.layers.43.mlp.experts.59.up_proj", "model.layers.43.mlp.experts.60.up_proj", "model.layers.43.mlp.experts.61.up_proj", "model.layers.43.mlp.experts.62.up_proj", "model.layers.43.mlp.experts.63.up_proj", "model.layers.43.mlp.experts.64.up_proj", "model.layers.43.mlp.experts.65.up_proj", "model.layers.43.mlp.experts.66.up_proj", "model.layers.43.mlp.experts.67.up_proj", "model.layers.43.mlp.experts.68.up_proj", "model.layers.43.mlp.experts.69.up_proj", "model.layers.43.mlp.experts.70.up_proj", "model.layers.43.mlp.experts.71.up_proj", "model.layers.43.mlp.experts.72.up_proj", "model.layers.43.mlp.experts.73.up_proj", "model.layers.43.mlp.experts.74.up_proj", "model.layers.43.mlp.experts.75.up_proj", "model.layers.43.mlp.experts.76.up_proj", "model.layers.43.mlp.experts.77.up_proj", "model.layers.43.mlp.experts.78.up_proj", "model.layers.43.mlp.experts.79.up_proj", "model.layers.43.mlp.experts.80.up_proj", "model.layers.43.mlp.experts.81.up_proj", "model.layers.43.mlp.experts.82.up_proj", "model.layers.43.mlp.experts.83.up_proj", "model.layers.43.mlp.experts.84.up_proj", "model.layers.43.mlp.experts.85.up_proj", "model.layers.43.mlp.experts.86.up_proj", "model.layers.43.mlp.experts.87.up_proj", "model.layers.43.mlp.experts.88.up_proj", "model.layers.43.mlp.experts.89.up_proj", "model.layers.43.mlp.experts.90.up_proj", "model.layers.43.mlp.experts.91.up_proj", "model.layers.43.mlp.experts.92.up_proj", "model.layers.43.mlp.experts.93.up_proj", "model.layers.43.mlp.experts.94.up_proj", "model.layers.43.mlp.experts.95.up_proj", "model.layers.43.mlp.experts.96.up_proj", "model.layers.43.mlp.experts.97.up_proj", "model.layers.43.mlp.experts.98.up_proj", "model.layers.43.mlp.experts.99.up_proj", "model.layers.43.mlp.experts.100.up_proj", "model.layers.43.mlp.experts.101.up_proj", "model.layers.43.mlp.experts.102.up_proj", "model.layers.43.mlp.experts.103.up_proj", "model.layers.43.mlp.experts.104.up_proj", "model.layers.43.mlp.experts.105.up_proj", "model.layers.43.mlp.experts.106.up_proj", "model.layers.43.mlp.experts.107.up_proj", "model.layers.43.mlp.experts.108.up_proj", "model.layers.43.mlp.experts.109.up_proj", "model.layers.43.mlp.experts.110.up_proj", "model.layers.43.mlp.experts.111.up_proj", "model.layers.43.mlp.experts.112.up_proj", "model.layers.43.mlp.experts.113.up_proj", "model.layers.43.mlp.experts.114.up_proj", "model.layers.43.mlp.experts.115.up_proj", "model.layers.43.mlp.experts.116.up_proj", "model.layers.43.mlp.experts.117.up_proj", "model.layers.43.mlp.experts.118.up_proj", "model.layers.43.mlp.experts.119.up_proj", "model.layers.43.mlp.experts.120.up_proj", "model.layers.43.mlp.experts.121.up_proj", "model.layers.43.mlp.experts.122.up_proj", "model.layers.43.mlp.experts.123.up_proj", "model.layers.43.mlp.experts.124.up_proj", "model.layers.43.mlp.experts.125.up_proj", "model.layers.43.mlp.experts.126.up_proj", "model.layers.43.mlp.experts.127.up_proj", "model.layers.43.mlp.experts.128.up_proj", "model.layers.43.mlp.experts.129.up_proj", "model.layers.43.mlp.experts.130.up_proj", "model.layers.43.mlp.experts.131.up_proj", "model.layers.43.mlp.experts.132.up_proj", "model.layers.43.mlp.experts.133.up_proj", "model.layers.43.mlp.experts.134.up_proj", "model.layers.43.mlp.experts.135.up_proj", "model.layers.43.mlp.experts.136.up_proj", "model.layers.43.mlp.experts.137.up_proj", "model.layers.43.mlp.experts.138.up_proj", "model.layers.43.mlp.experts.139.up_proj", "model.layers.43.mlp.experts.140.up_proj", "model.layers.43.mlp.experts.141.up_proj", "model.layers.43.mlp.experts.142.up_proj", "model.layers.43.mlp.experts.143.up_proj", "model.layers.43.mlp.experts.144.up_proj", "model.layers.43.mlp.experts.145.up_proj", "model.layers.43.mlp.experts.146.up_proj", "model.layers.43.mlp.experts.147.up_proj", "model.layers.43.mlp.experts.148.up_proj", "model.layers.43.mlp.experts.149.up_proj", "model.layers.43.mlp.experts.150.up_proj", "model.layers.43.mlp.experts.151.up_proj", "model.layers.43.mlp.experts.152.up_proj", "model.layers.43.mlp.experts.153.up_proj", "model.layers.43.mlp.experts.154.up_proj", "model.layers.43.mlp.experts.155.up_proj", "model.layers.43.mlp.experts.156.up_proj", "model.layers.43.mlp.experts.157.up_proj", "model.layers.43.mlp.experts.158.up_proj", "model.layers.43.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.415573832579044e-06, "dbits": 5033164800 } ] }, { "idx": 260, "layers": [ "model.layers.43.mlp.experts.0.down_proj", "model.layers.43.mlp.experts.1.down_proj", "model.layers.43.mlp.experts.2.down_proj", "model.layers.43.mlp.experts.3.down_proj", "model.layers.43.mlp.experts.4.down_proj", "model.layers.43.mlp.experts.5.down_proj", "model.layers.43.mlp.experts.6.down_proj", "model.layers.43.mlp.experts.7.down_proj", "model.layers.43.mlp.experts.8.down_proj", "model.layers.43.mlp.experts.9.down_proj", "model.layers.43.mlp.experts.10.down_proj", "model.layers.43.mlp.experts.11.down_proj", "model.layers.43.mlp.experts.12.down_proj", "model.layers.43.mlp.experts.13.down_proj", "model.layers.43.mlp.experts.14.down_proj", "model.layers.43.mlp.experts.15.down_proj", "model.layers.43.mlp.experts.16.down_proj", "model.layers.43.mlp.experts.17.down_proj", "model.layers.43.mlp.experts.18.down_proj", "model.layers.43.mlp.experts.19.down_proj", "model.layers.43.mlp.experts.20.down_proj", "model.layers.43.mlp.experts.21.down_proj", "model.layers.43.mlp.experts.22.down_proj", "model.layers.43.mlp.experts.23.down_proj", "model.layers.43.mlp.experts.24.down_proj", "model.layers.43.mlp.experts.25.down_proj", "model.layers.43.mlp.experts.26.down_proj", "model.layers.43.mlp.experts.27.down_proj", "model.layers.43.mlp.experts.28.down_proj", "model.layers.43.mlp.experts.29.down_proj", "model.layers.43.mlp.experts.30.down_proj", "model.layers.43.mlp.experts.31.down_proj", "model.layers.43.mlp.experts.32.down_proj", "model.layers.43.mlp.experts.33.down_proj", "model.layers.43.mlp.experts.34.down_proj", "model.layers.43.mlp.experts.35.down_proj", "model.layers.43.mlp.experts.36.down_proj", "model.layers.43.mlp.experts.37.down_proj", "model.layers.43.mlp.experts.38.down_proj", "model.layers.43.mlp.experts.39.down_proj", "model.layers.43.mlp.experts.40.down_proj", "model.layers.43.mlp.experts.41.down_proj", "model.layers.43.mlp.experts.42.down_proj", "model.layers.43.mlp.experts.43.down_proj", "model.layers.43.mlp.experts.44.down_proj", "model.layers.43.mlp.experts.45.down_proj", "model.layers.43.mlp.experts.46.down_proj", "model.layers.43.mlp.experts.47.down_proj", "model.layers.43.mlp.experts.48.down_proj", "model.layers.43.mlp.experts.49.down_proj", "model.layers.43.mlp.experts.50.down_proj", "model.layers.43.mlp.experts.51.down_proj", "model.layers.43.mlp.experts.52.down_proj", "model.layers.43.mlp.experts.53.down_proj", "model.layers.43.mlp.experts.54.down_proj", "model.layers.43.mlp.experts.55.down_proj", "model.layers.43.mlp.experts.56.down_proj", "model.layers.43.mlp.experts.57.down_proj", "model.layers.43.mlp.experts.58.down_proj", "model.layers.43.mlp.experts.59.down_proj", "model.layers.43.mlp.experts.60.down_proj", "model.layers.43.mlp.experts.61.down_proj", "model.layers.43.mlp.experts.62.down_proj", "model.layers.43.mlp.experts.63.down_proj", "model.layers.43.mlp.experts.64.down_proj", "model.layers.43.mlp.experts.65.down_proj", "model.layers.43.mlp.experts.66.down_proj", "model.layers.43.mlp.experts.67.down_proj", "model.layers.43.mlp.experts.68.down_proj", "model.layers.43.mlp.experts.69.down_proj", "model.layers.43.mlp.experts.70.down_proj", "model.layers.43.mlp.experts.71.down_proj", "model.layers.43.mlp.experts.72.down_proj", "model.layers.43.mlp.experts.73.down_proj", "model.layers.43.mlp.experts.74.down_proj", "model.layers.43.mlp.experts.75.down_proj", "model.layers.43.mlp.experts.76.down_proj", "model.layers.43.mlp.experts.77.down_proj", "model.layers.43.mlp.experts.78.down_proj", "model.layers.43.mlp.experts.79.down_proj", "model.layers.43.mlp.experts.80.down_proj", "model.layers.43.mlp.experts.81.down_proj", "model.layers.43.mlp.experts.82.down_proj", "model.layers.43.mlp.experts.83.down_proj", "model.layers.43.mlp.experts.84.down_proj", "model.layers.43.mlp.experts.85.down_proj", "model.layers.43.mlp.experts.86.down_proj", "model.layers.43.mlp.experts.87.down_proj", "model.layers.43.mlp.experts.88.down_proj", "model.layers.43.mlp.experts.89.down_proj", "model.layers.43.mlp.experts.90.down_proj", "model.layers.43.mlp.experts.91.down_proj", "model.layers.43.mlp.experts.92.down_proj", "model.layers.43.mlp.experts.93.down_proj", "model.layers.43.mlp.experts.94.down_proj", "model.layers.43.mlp.experts.95.down_proj", "model.layers.43.mlp.experts.96.down_proj", "model.layers.43.mlp.experts.97.down_proj", "model.layers.43.mlp.experts.98.down_proj", "model.layers.43.mlp.experts.99.down_proj", "model.layers.43.mlp.experts.100.down_proj", "model.layers.43.mlp.experts.101.down_proj", "model.layers.43.mlp.experts.102.down_proj", "model.layers.43.mlp.experts.103.down_proj", "model.layers.43.mlp.experts.104.down_proj", "model.layers.43.mlp.experts.105.down_proj", "model.layers.43.mlp.experts.106.down_proj", "model.layers.43.mlp.experts.107.down_proj", "model.layers.43.mlp.experts.108.down_proj", "model.layers.43.mlp.experts.109.down_proj", "model.layers.43.mlp.experts.110.down_proj", "model.layers.43.mlp.experts.111.down_proj", "model.layers.43.mlp.experts.112.down_proj", "model.layers.43.mlp.experts.113.down_proj", "model.layers.43.mlp.experts.114.down_proj", "model.layers.43.mlp.experts.115.down_proj", "model.layers.43.mlp.experts.116.down_proj", "model.layers.43.mlp.experts.117.down_proj", "model.layers.43.mlp.experts.118.down_proj", "model.layers.43.mlp.experts.119.down_proj", "model.layers.43.mlp.experts.120.down_proj", "model.layers.43.mlp.experts.121.down_proj", "model.layers.43.mlp.experts.122.down_proj", "model.layers.43.mlp.experts.123.down_proj", "model.layers.43.mlp.experts.124.down_proj", "model.layers.43.mlp.experts.125.down_proj", "model.layers.43.mlp.experts.126.down_proj", "model.layers.43.mlp.experts.127.down_proj", "model.layers.43.mlp.experts.128.down_proj", "model.layers.43.mlp.experts.129.down_proj", "model.layers.43.mlp.experts.130.down_proj", "model.layers.43.mlp.experts.131.down_proj", "model.layers.43.mlp.experts.132.down_proj", "model.layers.43.mlp.experts.133.down_proj", "model.layers.43.mlp.experts.134.down_proj", "model.layers.43.mlp.experts.135.down_proj", "model.layers.43.mlp.experts.136.down_proj", "model.layers.43.mlp.experts.137.down_proj", "model.layers.43.mlp.experts.138.down_proj", "model.layers.43.mlp.experts.139.down_proj", "model.layers.43.mlp.experts.140.down_proj", "model.layers.43.mlp.experts.141.down_proj", "model.layers.43.mlp.experts.142.down_proj", "model.layers.43.mlp.experts.143.down_proj", "model.layers.43.mlp.experts.144.down_proj", "model.layers.43.mlp.experts.145.down_proj", "model.layers.43.mlp.experts.146.down_proj", "model.layers.43.mlp.experts.147.down_proj", "model.layers.43.mlp.experts.148.down_proj", "model.layers.43.mlp.experts.149.down_proj", "model.layers.43.mlp.experts.150.down_proj", "model.layers.43.mlp.experts.151.down_proj", "model.layers.43.mlp.experts.152.down_proj", "model.layers.43.mlp.experts.153.down_proj", "model.layers.43.mlp.experts.154.down_proj", "model.layers.43.mlp.experts.155.down_proj", "model.layers.43.mlp.experts.156.down_proj", "model.layers.43.mlp.experts.157.down_proj", "model.layers.43.mlp.experts.158.down_proj", "model.layers.43.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.2428728193040326e-07, "dbits": 2516582400 } ] }, { "idx": 261, "layers": [ "model.layers.44.self_attn.q_proj" ], "candidates": [ { "dkld": 2.9100774554535747e-06, "dbits": 125829120 } ] }, { "idx": 262, "layers": [ "model.layers.44.self_attn.k_proj", "model.layers.44.self_attn.v_proj" ], "candidates": [ { "dkld": 1.019565388560295e-06, "dbits": 20971520 } ] }, { "idx": 263, "layers": [ "model.layers.44.self_attn.o_proj" ], "candidates": [ { "dkld": 3.6217359593137665e-06, "dbits": 125829120 } ] }, { "idx": 264, "layers": [ "model.layers.44.mlp.shared_experts.gate_proj", "model.layers.44.mlp.shared_experts.up_proj", "model.layers.44.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.752701402641876e-06, "dbits": 47185920 } ] }, { "idx": 265, "layers": [ "model.layers.44.mlp.experts.0.gate_proj", "model.layers.44.mlp.experts.1.gate_proj", "model.layers.44.mlp.experts.2.gate_proj", "model.layers.44.mlp.experts.3.gate_proj", "model.layers.44.mlp.experts.4.gate_proj", "model.layers.44.mlp.experts.5.gate_proj", "model.layers.44.mlp.experts.6.gate_proj", "model.layers.44.mlp.experts.7.gate_proj", "model.layers.44.mlp.experts.8.gate_proj", "model.layers.44.mlp.experts.9.gate_proj", "model.layers.44.mlp.experts.10.gate_proj", "model.layers.44.mlp.experts.11.gate_proj", "model.layers.44.mlp.experts.12.gate_proj", "model.layers.44.mlp.experts.13.gate_proj", "model.layers.44.mlp.experts.14.gate_proj", "model.layers.44.mlp.experts.15.gate_proj", "model.layers.44.mlp.experts.16.gate_proj", "model.layers.44.mlp.experts.17.gate_proj", "model.layers.44.mlp.experts.18.gate_proj", "model.layers.44.mlp.experts.19.gate_proj", "model.layers.44.mlp.experts.20.gate_proj", "model.layers.44.mlp.experts.21.gate_proj", "model.layers.44.mlp.experts.22.gate_proj", "model.layers.44.mlp.experts.23.gate_proj", "model.layers.44.mlp.experts.24.gate_proj", "model.layers.44.mlp.experts.25.gate_proj", "model.layers.44.mlp.experts.26.gate_proj", "model.layers.44.mlp.experts.27.gate_proj", "model.layers.44.mlp.experts.28.gate_proj", "model.layers.44.mlp.experts.29.gate_proj", "model.layers.44.mlp.experts.30.gate_proj", "model.layers.44.mlp.experts.31.gate_proj", "model.layers.44.mlp.experts.32.gate_proj", "model.layers.44.mlp.experts.33.gate_proj", "model.layers.44.mlp.experts.34.gate_proj", "model.layers.44.mlp.experts.35.gate_proj", "model.layers.44.mlp.experts.36.gate_proj", "model.layers.44.mlp.experts.37.gate_proj", "model.layers.44.mlp.experts.38.gate_proj", "model.layers.44.mlp.experts.39.gate_proj", "model.layers.44.mlp.experts.40.gate_proj", "model.layers.44.mlp.experts.41.gate_proj", "model.layers.44.mlp.experts.42.gate_proj", "model.layers.44.mlp.experts.43.gate_proj", "model.layers.44.mlp.experts.44.gate_proj", "model.layers.44.mlp.experts.45.gate_proj", "model.layers.44.mlp.experts.46.gate_proj", "model.layers.44.mlp.experts.47.gate_proj", "model.layers.44.mlp.experts.48.gate_proj", "model.layers.44.mlp.experts.49.gate_proj", "model.layers.44.mlp.experts.50.gate_proj", "model.layers.44.mlp.experts.51.gate_proj", "model.layers.44.mlp.experts.52.gate_proj", "model.layers.44.mlp.experts.53.gate_proj", "model.layers.44.mlp.experts.54.gate_proj", "model.layers.44.mlp.experts.55.gate_proj", "model.layers.44.mlp.experts.56.gate_proj", "model.layers.44.mlp.experts.57.gate_proj", "model.layers.44.mlp.experts.58.gate_proj", "model.layers.44.mlp.experts.59.gate_proj", "model.layers.44.mlp.experts.60.gate_proj", "model.layers.44.mlp.experts.61.gate_proj", "model.layers.44.mlp.experts.62.gate_proj", "model.layers.44.mlp.experts.63.gate_proj", "model.layers.44.mlp.experts.64.gate_proj", "model.layers.44.mlp.experts.65.gate_proj", "model.layers.44.mlp.experts.66.gate_proj", "model.layers.44.mlp.experts.67.gate_proj", "model.layers.44.mlp.experts.68.gate_proj", "model.layers.44.mlp.experts.69.gate_proj", "model.layers.44.mlp.experts.70.gate_proj", "model.layers.44.mlp.experts.71.gate_proj", "model.layers.44.mlp.experts.72.gate_proj", "model.layers.44.mlp.experts.73.gate_proj", "model.layers.44.mlp.experts.74.gate_proj", "model.layers.44.mlp.experts.75.gate_proj", "model.layers.44.mlp.experts.76.gate_proj", "model.layers.44.mlp.experts.77.gate_proj", "model.layers.44.mlp.experts.78.gate_proj", "model.layers.44.mlp.experts.79.gate_proj", "model.layers.44.mlp.experts.80.gate_proj", "model.layers.44.mlp.experts.81.gate_proj", "model.layers.44.mlp.experts.82.gate_proj", "model.layers.44.mlp.experts.83.gate_proj", "model.layers.44.mlp.experts.84.gate_proj", "model.layers.44.mlp.experts.85.gate_proj", "model.layers.44.mlp.experts.86.gate_proj", "model.layers.44.mlp.experts.87.gate_proj", "model.layers.44.mlp.experts.88.gate_proj", "model.layers.44.mlp.experts.89.gate_proj", "model.layers.44.mlp.experts.90.gate_proj", "model.layers.44.mlp.experts.91.gate_proj", "model.layers.44.mlp.experts.92.gate_proj", "model.layers.44.mlp.experts.93.gate_proj", "model.layers.44.mlp.experts.94.gate_proj", "model.layers.44.mlp.experts.95.gate_proj", "model.layers.44.mlp.experts.96.gate_proj", "model.layers.44.mlp.experts.97.gate_proj", "model.layers.44.mlp.experts.98.gate_proj", "model.layers.44.mlp.experts.99.gate_proj", "model.layers.44.mlp.experts.100.gate_proj", "model.layers.44.mlp.experts.101.gate_proj", "model.layers.44.mlp.experts.102.gate_proj", "model.layers.44.mlp.experts.103.gate_proj", "model.layers.44.mlp.experts.104.gate_proj", "model.layers.44.mlp.experts.105.gate_proj", "model.layers.44.mlp.experts.106.gate_proj", "model.layers.44.mlp.experts.107.gate_proj", "model.layers.44.mlp.experts.108.gate_proj", "model.layers.44.mlp.experts.109.gate_proj", "model.layers.44.mlp.experts.110.gate_proj", "model.layers.44.mlp.experts.111.gate_proj", "model.layers.44.mlp.experts.112.gate_proj", "model.layers.44.mlp.experts.113.gate_proj", "model.layers.44.mlp.experts.114.gate_proj", "model.layers.44.mlp.experts.115.gate_proj", "model.layers.44.mlp.experts.116.gate_proj", "model.layers.44.mlp.experts.117.gate_proj", "model.layers.44.mlp.experts.118.gate_proj", "model.layers.44.mlp.experts.119.gate_proj", "model.layers.44.mlp.experts.120.gate_proj", "model.layers.44.mlp.experts.121.gate_proj", "model.layers.44.mlp.experts.122.gate_proj", "model.layers.44.mlp.experts.123.gate_proj", "model.layers.44.mlp.experts.124.gate_proj", "model.layers.44.mlp.experts.125.gate_proj", "model.layers.44.mlp.experts.126.gate_proj", "model.layers.44.mlp.experts.127.gate_proj", "model.layers.44.mlp.experts.128.gate_proj", "model.layers.44.mlp.experts.129.gate_proj", "model.layers.44.mlp.experts.130.gate_proj", "model.layers.44.mlp.experts.131.gate_proj", "model.layers.44.mlp.experts.132.gate_proj", "model.layers.44.mlp.experts.133.gate_proj", "model.layers.44.mlp.experts.134.gate_proj", "model.layers.44.mlp.experts.135.gate_proj", "model.layers.44.mlp.experts.136.gate_proj", "model.layers.44.mlp.experts.137.gate_proj", "model.layers.44.mlp.experts.138.gate_proj", "model.layers.44.mlp.experts.139.gate_proj", "model.layers.44.mlp.experts.140.gate_proj", "model.layers.44.mlp.experts.141.gate_proj", "model.layers.44.mlp.experts.142.gate_proj", "model.layers.44.mlp.experts.143.gate_proj", "model.layers.44.mlp.experts.144.gate_proj", "model.layers.44.mlp.experts.145.gate_proj", "model.layers.44.mlp.experts.146.gate_proj", "model.layers.44.mlp.experts.147.gate_proj", "model.layers.44.mlp.experts.148.gate_proj", "model.layers.44.mlp.experts.149.gate_proj", "model.layers.44.mlp.experts.150.gate_proj", "model.layers.44.mlp.experts.151.gate_proj", "model.layers.44.mlp.experts.152.gate_proj", "model.layers.44.mlp.experts.153.gate_proj", "model.layers.44.mlp.experts.154.gate_proj", "model.layers.44.mlp.experts.155.gate_proj", "model.layers.44.mlp.experts.156.gate_proj", "model.layers.44.mlp.experts.157.gate_proj", "model.layers.44.mlp.experts.158.gate_proj", "model.layers.44.mlp.experts.159.gate_proj", "model.layers.44.mlp.experts.0.up_proj", "model.layers.44.mlp.experts.1.up_proj", "model.layers.44.mlp.experts.2.up_proj", "model.layers.44.mlp.experts.3.up_proj", "model.layers.44.mlp.experts.4.up_proj", "model.layers.44.mlp.experts.5.up_proj", "model.layers.44.mlp.experts.6.up_proj", "model.layers.44.mlp.experts.7.up_proj", "model.layers.44.mlp.experts.8.up_proj", "model.layers.44.mlp.experts.9.up_proj", "model.layers.44.mlp.experts.10.up_proj", "model.layers.44.mlp.experts.11.up_proj", "model.layers.44.mlp.experts.12.up_proj", "model.layers.44.mlp.experts.13.up_proj", "model.layers.44.mlp.experts.14.up_proj", "model.layers.44.mlp.experts.15.up_proj", "model.layers.44.mlp.experts.16.up_proj", "model.layers.44.mlp.experts.17.up_proj", "model.layers.44.mlp.experts.18.up_proj", "model.layers.44.mlp.experts.19.up_proj", "model.layers.44.mlp.experts.20.up_proj", "model.layers.44.mlp.experts.21.up_proj", "model.layers.44.mlp.experts.22.up_proj", "model.layers.44.mlp.experts.23.up_proj", "model.layers.44.mlp.experts.24.up_proj", "model.layers.44.mlp.experts.25.up_proj", "model.layers.44.mlp.experts.26.up_proj", "model.layers.44.mlp.experts.27.up_proj", "model.layers.44.mlp.experts.28.up_proj", "model.layers.44.mlp.experts.29.up_proj", "model.layers.44.mlp.experts.30.up_proj", "model.layers.44.mlp.experts.31.up_proj", "model.layers.44.mlp.experts.32.up_proj", "model.layers.44.mlp.experts.33.up_proj", "model.layers.44.mlp.experts.34.up_proj", "model.layers.44.mlp.experts.35.up_proj", "model.layers.44.mlp.experts.36.up_proj", "model.layers.44.mlp.experts.37.up_proj", "model.layers.44.mlp.experts.38.up_proj", "model.layers.44.mlp.experts.39.up_proj", "model.layers.44.mlp.experts.40.up_proj", "model.layers.44.mlp.experts.41.up_proj", "model.layers.44.mlp.experts.42.up_proj", "model.layers.44.mlp.experts.43.up_proj", "model.layers.44.mlp.experts.44.up_proj", "model.layers.44.mlp.experts.45.up_proj", "model.layers.44.mlp.experts.46.up_proj", "model.layers.44.mlp.experts.47.up_proj", "model.layers.44.mlp.experts.48.up_proj", "model.layers.44.mlp.experts.49.up_proj", "model.layers.44.mlp.experts.50.up_proj", "model.layers.44.mlp.experts.51.up_proj", "model.layers.44.mlp.experts.52.up_proj", "model.layers.44.mlp.experts.53.up_proj", "model.layers.44.mlp.experts.54.up_proj", "model.layers.44.mlp.experts.55.up_proj", "model.layers.44.mlp.experts.56.up_proj", "model.layers.44.mlp.experts.57.up_proj", "model.layers.44.mlp.experts.58.up_proj", "model.layers.44.mlp.experts.59.up_proj", "model.layers.44.mlp.experts.60.up_proj", "model.layers.44.mlp.experts.61.up_proj", "model.layers.44.mlp.experts.62.up_proj", "model.layers.44.mlp.experts.63.up_proj", "model.layers.44.mlp.experts.64.up_proj", "model.layers.44.mlp.experts.65.up_proj", "model.layers.44.mlp.experts.66.up_proj", "model.layers.44.mlp.experts.67.up_proj", "model.layers.44.mlp.experts.68.up_proj", "model.layers.44.mlp.experts.69.up_proj", "model.layers.44.mlp.experts.70.up_proj", "model.layers.44.mlp.experts.71.up_proj", "model.layers.44.mlp.experts.72.up_proj", "model.layers.44.mlp.experts.73.up_proj", "model.layers.44.mlp.experts.74.up_proj", "model.layers.44.mlp.experts.75.up_proj", "model.layers.44.mlp.experts.76.up_proj", "model.layers.44.mlp.experts.77.up_proj", "model.layers.44.mlp.experts.78.up_proj", "model.layers.44.mlp.experts.79.up_proj", "model.layers.44.mlp.experts.80.up_proj", "model.layers.44.mlp.experts.81.up_proj", "model.layers.44.mlp.experts.82.up_proj", "model.layers.44.mlp.experts.83.up_proj", "model.layers.44.mlp.experts.84.up_proj", "model.layers.44.mlp.experts.85.up_proj", "model.layers.44.mlp.experts.86.up_proj", "model.layers.44.mlp.experts.87.up_proj", "model.layers.44.mlp.experts.88.up_proj", "model.layers.44.mlp.experts.89.up_proj", "model.layers.44.mlp.experts.90.up_proj", "model.layers.44.mlp.experts.91.up_proj", "model.layers.44.mlp.experts.92.up_proj", "model.layers.44.mlp.experts.93.up_proj", "model.layers.44.mlp.experts.94.up_proj", "model.layers.44.mlp.experts.95.up_proj", "model.layers.44.mlp.experts.96.up_proj", "model.layers.44.mlp.experts.97.up_proj", "model.layers.44.mlp.experts.98.up_proj", "model.layers.44.mlp.experts.99.up_proj", "model.layers.44.mlp.experts.100.up_proj", "model.layers.44.mlp.experts.101.up_proj", "model.layers.44.mlp.experts.102.up_proj", "model.layers.44.mlp.experts.103.up_proj", "model.layers.44.mlp.experts.104.up_proj", "model.layers.44.mlp.experts.105.up_proj", "model.layers.44.mlp.experts.106.up_proj", "model.layers.44.mlp.experts.107.up_proj", "model.layers.44.mlp.experts.108.up_proj", "model.layers.44.mlp.experts.109.up_proj", "model.layers.44.mlp.experts.110.up_proj", "model.layers.44.mlp.experts.111.up_proj", "model.layers.44.mlp.experts.112.up_proj", "model.layers.44.mlp.experts.113.up_proj", "model.layers.44.mlp.experts.114.up_proj", "model.layers.44.mlp.experts.115.up_proj", "model.layers.44.mlp.experts.116.up_proj", "model.layers.44.mlp.experts.117.up_proj", "model.layers.44.mlp.experts.118.up_proj", "model.layers.44.mlp.experts.119.up_proj", "model.layers.44.mlp.experts.120.up_proj", "model.layers.44.mlp.experts.121.up_proj", "model.layers.44.mlp.experts.122.up_proj", "model.layers.44.mlp.experts.123.up_proj", "model.layers.44.mlp.experts.124.up_proj", "model.layers.44.mlp.experts.125.up_proj", "model.layers.44.mlp.experts.126.up_proj", "model.layers.44.mlp.experts.127.up_proj", "model.layers.44.mlp.experts.128.up_proj", "model.layers.44.mlp.experts.129.up_proj", "model.layers.44.mlp.experts.130.up_proj", "model.layers.44.mlp.experts.131.up_proj", "model.layers.44.mlp.experts.132.up_proj", "model.layers.44.mlp.experts.133.up_proj", "model.layers.44.mlp.experts.134.up_proj", "model.layers.44.mlp.experts.135.up_proj", "model.layers.44.mlp.experts.136.up_proj", "model.layers.44.mlp.experts.137.up_proj", "model.layers.44.mlp.experts.138.up_proj", "model.layers.44.mlp.experts.139.up_proj", "model.layers.44.mlp.experts.140.up_proj", "model.layers.44.mlp.experts.141.up_proj", "model.layers.44.mlp.experts.142.up_proj", "model.layers.44.mlp.experts.143.up_proj", "model.layers.44.mlp.experts.144.up_proj", "model.layers.44.mlp.experts.145.up_proj", "model.layers.44.mlp.experts.146.up_proj", "model.layers.44.mlp.experts.147.up_proj", "model.layers.44.mlp.experts.148.up_proj", "model.layers.44.mlp.experts.149.up_proj", "model.layers.44.mlp.experts.150.up_proj", "model.layers.44.mlp.experts.151.up_proj", "model.layers.44.mlp.experts.152.up_proj", "model.layers.44.mlp.experts.153.up_proj", "model.layers.44.mlp.experts.154.up_proj", "model.layers.44.mlp.experts.155.up_proj", "model.layers.44.mlp.experts.156.up_proj", "model.layers.44.mlp.experts.157.up_proj", "model.layers.44.mlp.experts.158.up_proj", "model.layers.44.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.685365663841345e-06, "dbits": 5033164800 } ] }, { "idx": 266, "layers": [ "model.layers.44.mlp.experts.0.down_proj", "model.layers.44.mlp.experts.1.down_proj", "model.layers.44.mlp.experts.2.down_proj", "model.layers.44.mlp.experts.3.down_proj", "model.layers.44.mlp.experts.4.down_proj", "model.layers.44.mlp.experts.5.down_proj", "model.layers.44.mlp.experts.6.down_proj", "model.layers.44.mlp.experts.7.down_proj", "model.layers.44.mlp.experts.8.down_proj", "model.layers.44.mlp.experts.9.down_proj", "model.layers.44.mlp.experts.10.down_proj", "model.layers.44.mlp.experts.11.down_proj", "model.layers.44.mlp.experts.12.down_proj", "model.layers.44.mlp.experts.13.down_proj", "model.layers.44.mlp.experts.14.down_proj", "model.layers.44.mlp.experts.15.down_proj", "model.layers.44.mlp.experts.16.down_proj", "model.layers.44.mlp.experts.17.down_proj", "model.layers.44.mlp.experts.18.down_proj", "model.layers.44.mlp.experts.19.down_proj", "model.layers.44.mlp.experts.20.down_proj", "model.layers.44.mlp.experts.21.down_proj", "model.layers.44.mlp.experts.22.down_proj", "model.layers.44.mlp.experts.23.down_proj", "model.layers.44.mlp.experts.24.down_proj", "model.layers.44.mlp.experts.25.down_proj", "model.layers.44.mlp.experts.26.down_proj", "model.layers.44.mlp.experts.27.down_proj", "model.layers.44.mlp.experts.28.down_proj", "model.layers.44.mlp.experts.29.down_proj", "model.layers.44.mlp.experts.30.down_proj", "model.layers.44.mlp.experts.31.down_proj", "model.layers.44.mlp.experts.32.down_proj", "model.layers.44.mlp.experts.33.down_proj", "model.layers.44.mlp.experts.34.down_proj", "model.layers.44.mlp.experts.35.down_proj", "model.layers.44.mlp.experts.36.down_proj", "model.layers.44.mlp.experts.37.down_proj", "model.layers.44.mlp.experts.38.down_proj", "model.layers.44.mlp.experts.39.down_proj", "model.layers.44.mlp.experts.40.down_proj", "model.layers.44.mlp.experts.41.down_proj", "model.layers.44.mlp.experts.42.down_proj", "model.layers.44.mlp.experts.43.down_proj", "model.layers.44.mlp.experts.44.down_proj", "model.layers.44.mlp.experts.45.down_proj", "model.layers.44.mlp.experts.46.down_proj", "model.layers.44.mlp.experts.47.down_proj", "model.layers.44.mlp.experts.48.down_proj", "model.layers.44.mlp.experts.49.down_proj", "model.layers.44.mlp.experts.50.down_proj", "model.layers.44.mlp.experts.51.down_proj", "model.layers.44.mlp.experts.52.down_proj", "model.layers.44.mlp.experts.53.down_proj", "model.layers.44.mlp.experts.54.down_proj", "model.layers.44.mlp.experts.55.down_proj", "model.layers.44.mlp.experts.56.down_proj", "model.layers.44.mlp.experts.57.down_proj", "model.layers.44.mlp.experts.58.down_proj", "model.layers.44.mlp.experts.59.down_proj", "model.layers.44.mlp.experts.60.down_proj", "model.layers.44.mlp.experts.61.down_proj", "model.layers.44.mlp.experts.62.down_proj", "model.layers.44.mlp.experts.63.down_proj", "model.layers.44.mlp.experts.64.down_proj", "model.layers.44.mlp.experts.65.down_proj", "model.layers.44.mlp.experts.66.down_proj", "model.layers.44.mlp.experts.67.down_proj", "model.layers.44.mlp.experts.68.down_proj", "model.layers.44.mlp.experts.69.down_proj", "model.layers.44.mlp.experts.70.down_proj", "model.layers.44.mlp.experts.71.down_proj", "model.layers.44.mlp.experts.72.down_proj", "model.layers.44.mlp.experts.73.down_proj", "model.layers.44.mlp.experts.74.down_proj", "model.layers.44.mlp.experts.75.down_proj", "model.layers.44.mlp.experts.76.down_proj", "model.layers.44.mlp.experts.77.down_proj", "model.layers.44.mlp.experts.78.down_proj", "model.layers.44.mlp.experts.79.down_proj", "model.layers.44.mlp.experts.80.down_proj", "model.layers.44.mlp.experts.81.down_proj", "model.layers.44.mlp.experts.82.down_proj", "model.layers.44.mlp.experts.83.down_proj", "model.layers.44.mlp.experts.84.down_proj", "model.layers.44.mlp.experts.85.down_proj", "model.layers.44.mlp.experts.86.down_proj", "model.layers.44.mlp.experts.87.down_proj", "model.layers.44.mlp.experts.88.down_proj", "model.layers.44.mlp.experts.89.down_proj", "model.layers.44.mlp.experts.90.down_proj", "model.layers.44.mlp.experts.91.down_proj", "model.layers.44.mlp.experts.92.down_proj", "model.layers.44.mlp.experts.93.down_proj", "model.layers.44.mlp.experts.94.down_proj", "model.layers.44.mlp.experts.95.down_proj", "model.layers.44.mlp.experts.96.down_proj", "model.layers.44.mlp.experts.97.down_proj", "model.layers.44.mlp.experts.98.down_proj", "model.layers.44.mlp.experts.99.down_proj", "model.layers.44.mlp.experts.100.down_proj", "model.layers.44.mlp.experts.101.down_proj", "model.layers.44.mlp.experts.102.down_proj", "model.layers.44.mlp.experts.103.down_proj", "model.layers.44.mlp.experts.104.down_proj", "model.layers.44.mlp.experts.105.down_proj", "model.layers.44.mlp.experts.106.down_proj", "model.layers.44.mlp.experts.107.down_proj", "model.layers.44.mlp.experts.108.down_proj", "model.layers.44.mlp.experts.109.down_proj", "model.layers.44.mlp.experts.110.down_proj", "model.layers.44.mlp.experts.111.down_proj", "model.layers.44.mlp.experts.112.down_proj", "model.layers.44.mlp.experts.113.down_proj", "model.layers.44.mlp.experts.114.down_proj", "model.layers.44.mlp.experts.115.down_proj", "model.layers.44.mlp.experts.116.down_proj", "model.layers.44.mlp.experts.117.down_proj", "model.layers.44.mlp.experts.118.down_proj", "model.layers.44.mlp.experts.119.down_proj", "model.layers.44.mlp.experts.120.down_proj", "model.layers.44.mlp.experts.121.down_proj", "model.layers.44.mlp.experts.122.down_proj", "model.layers.44.mlp.experts.123.down_proj", "model.layers.44.mlp.experts.124.down_proj", "model.layers.44.mlp.experts.125.down_proj", "model.layers.44.mlp.experts.126.down_proj", "model.layers.44.mlp.experts.127.down_proj", "model.layers.44.mlp.experts.128.down_proj", "model.layers.44.mlp.experts.129.down_proj", "model.layers.44.mlp.experts.130.down_proj", "model.layers.44.mlp.experts.131.down_proj", "model.layers.44.mlp.experts.132.down_proj", "model.layers.44.mlp.experts.133.down_proj", "model.layers.44.mlp.experts.134.down_proj", "model.layers.44.mlp.experts.135.down_proj", "model.layers.44.mlp.experts.136.down_proj", "model.layers.44.mlp.experts.137.down_proj", "model.layers.44.mlp.experts.138.down_proj", "model.layers.44.mlp.experts.139.down_proj", "model.layers.44.mlp.experts.140.down_proj", "model.layers.44.mlp.experts.141.down_proj", "model.layers.44.mlp.experts.142.down_proj", "model.layers.44.mlp.experts.143.down_proj", "model.layers.44.mlp.experts.144.down_proj", "model.layers.44.mlp.experts.145.down_proj", "model.layers.44.mlp.experts.146.down_proj", "model.layers.44.mlp.experts.147.down_proj", "model.layers.44.mlp.experts.148.down_proj", "model.layers.44.mlp.experts.149.down_proj", "model.layers.44.mlp.experts.150.down_proj", "model.layers.44.mlp.experts.151.down_proj", "model.layers.44.mlp.experts.152.down_proj", "model.layers.44.mlp.experts.153.down_proj", "model.layers.44.mlp.experts.154.down_proj", "model.layers.44.mlp.experts.155.down_proj", "model.layers.44.mlp.experts.156.down_proj", "model.layers.44.mlp.experts.157.down_proj", "model.layers.44.mlp.experts.158.down_proj", "model.layers.44.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.0961579391732595e-06, "dbits": 2516582400 } ] }, { "idx": 267, "layers": [ "model.layers.45.self_attn.q_proj" ], "candidates": [ { "dkld": -2.756150206550999e-06, "dbits": 125829120 } ] }, { "idx": 268, "layers": [ "model.layers.45.self_attn.k_proj", "model.layers.45.self_attn.v_proj" ], "candidates": [ { "dkld": 3.036661655642049e-06, "dbits": 20971520 } ] }, { "idx": 269, "layers": [ "model.layers.45.self_attn.o_proj" ], "candidates": [ { "dkld": -6.502930773421688e-07, "dbits": 125829120 } ] }, { "idx": 270, "layers": [ "model.layers.45.mlp.shared_experts.gate_proj", "model.layers.45.mlp.shared_experts.up_proj", "model.layers.45.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.1334137525409677e-06, "dbits": 47185920 } ] }, { "idx": 271, "layers": [ "model.layers.45.mlp.experts.0.gate_proj", "model.layers.45.mlp.experts.1.gate_proj", "model.layers.45.mlp.experts.2.gate_proj", "model.layers.45.mlp.experts.3.gate_proj", "model.layers.45.mlp.experts.4.gate_proj", "model.layers.45.mlp.experts.5.gate_proj", "model.layers.45.mlp.experts.6.gate_proj", "model.layers.45.mlp.experts.7.gate_proj", "model.layers.45.mlp.experts.8.gate_proj", "model.layers.45.mlp.experts.9.gate_proj", "model.layers.45.mlp.experts.10.gate_proj", "model.layers.45.mlp.experts.11.gate_proj", "model.layers.45.mlp.experts.12.gate_proj", "model.layers.45.mlp.experts.13.gate_proj", "model.layers.45.mlp.experts.14.gate_proj", "model.layers.45.mlp.experts.15.gate_proj", "model.layers.45.mlp.experts.16.gate_proj", "model.layers.45.mlp.experts.17.gate_proj", "model.layers.45.mlp.experts.18.gate_proj", "model.layers.45.mlp.experts.19.gate_proj", "model.layers.45.mlp.experts.20.gate_proj", "model.layers.45.mlp.experts.21.gate_proj", "model.layers.45.mlp.experts.22.gate_proj", "model.layers.45.mlp.experts.23.gate_proj", "model.layers.45.mlp.experts.24.gate_proj", "model.layers.45.mlp.experts.25.gate_proj", "model.layers.45.mlp.experts.26.gate_proj", "model.layers.45.mlp.experts.27.gate_proj", "model.layers.45.mlp.experts.28.gate_proj", "model.layers.45.mlp.experts.29.gate_proj", "model.layers.45.mlp.experts.30.gate_proj", "model.layers.45.mlp.experts.31.gate_proj", "model.layers.45.mlp.experts.32.gate_proj", "model.layers.45.mlp.experts.33.gate_proj", "model.layers.45.mlp.experts.34.gate_proj", "model.layers.45.mlp.experts.35.gate_proj", "model.layers.45.mlp.experts.36.gate_proj", "model.layers.45.mlp.experts.37.gate_proj", "model.layers.45.mlp.experts.38.gate_proj", "model.layers.45.mlp.experts.39.gate_proj", "model.layers.45.mlp.experts.40.gate_proj", "model.layers.45.mlp.experts.41.gate_proj", "model.layers.45.mlp.experts.42.gate_proj", "model.layers.45.mlp.experts.43.gate_proj", "model.layers.45.mlp.experts.44.gate_proj", "model.layers.45.mlp.experts.45.gate_proj", "model.layers.45.mlp.experts.46.gate_proj", "model.layers.45.mlp.experts.47.gate_proj", "model.layers.45.mlp.experts.48.gate_proj", "model.layers.45.mlp.experts.49.gate_proj", "model.layers.45.mlp.experts.50.gate_proj", "model.layers.45.mlp.experts.51.gate_proj", "model.layers.45.mlp.experts.52.gate_proj", "model.layers.45.mlp.experts.53.gate_proj", "model.layers.45.mlp.experts.54.gate_proj", "model.layers.45.mlp.experts.55.gate_proj", "model.layers.45.mlp.experts.56.gate_proj", "model.layers.45.mlp.experts.57.gate_proj", "model.layers.45.mlp.experts.58.gate_proj", "model.layers.45.mlp.experts.59.gate_proj", "model.layers.45.mlp.experts.60.gate_proj", "model.layers.45.mlp.experts.61.gate_proj", "model.layers.45.mlp.experts.62.gate_proj", "model.layers.45.mlp.experts.63.gate_proj", "model.layers.45.mlp.experts.64.gate_proj", "model.layers.45.mlp.experts.65.gate_proj", "model.layers.45.mlp.experts.66.gate_proj", "model.layers.45.mlp.experts.67.gate_proj", "model.layers.45.mlp.experts.68.gate_proj", "model.layers.45.mlp.experts.69.gate_proj", "model.layers.45.mlp.experts.70.gate_proj", "model.layers.45.mlp.experts.71.gate_proj", "model.layers.45.mlp.experts.72.gate_proj", "model.layers.45.mlp.experts.73.gate_proj", "model.layers.45.mlp.experts.74.gate_proj", "model.layers.45.mlp.experts.75.gate_proj", "model.layers.45.mlp.experts.76.gate_proj", "model.layers.45.mlp.experts.77.gate_proj", "model.layers.45.mlp.experts.78.gate_proj", "model.layers.45.mlp.experts.79.gate_proj", "model.layers.45.mlp.experts.80.gate_proj", "model.layers.45.mlp.experts.81.gate_proj", "model.layers.45.mlp.experts.82.gate_proj", "model.layers.45.mlp.experts.83.gate_proj", "model.layers.45.mlp.experts.84.gate_proj", "model.layers.45.mlp.experts.85.gate_proj", "model.layers.45.mlp.experts.86.gate_proj", "model.layers.45.mlp.experts.87.gate_proj", "model.layers.45.mlp.experts.88.gate_proj", "model.layers.45.mlp.experts.89.gate_proj", "model.layers.45.mlp.experts.90.gate_proj", "model.layers.45.mlp.experts.91.gate_proj", "model.layers.45.mlp.experts.92.gate_proj", "model.layers.45.mlp.experts.93.gate_proj", "model.layers.45.mlp.experts.94.gate_proj", "model.layers.45.mlp.experts.95.gate_proj", "model.layers.45.mlp.experts.96.gate_proj", "model.layers.45.mlp.experts.97.gate_proj", "model.layers.45.mlp.experts.98.gate_proj", "model.layers.45.mlp.experts.99.gate_proj", "model.layers.45.mlp.experts.100.gate_proj", "model.layers.45.mlp.experts.101.gate_proj", "model.layers.45.mlp.experts.102.gate_proj", "model.layers.45.mlp.experts.103.gate_proj", "model.layers.45.mlp.experts.104.gate_proj", "model.layers.45.mlp.experts.105.gate_proj", "model.layers.45.mlp.experts.106.gate_proj", "model.layers.45.mlp.experts.107.gate_proj", "model.layers.45.mlp.experts.108.gate_proj", "model.layers.45.mlp.experts.109.gate_proj", "model.layers.45.mlp.experts.110.gate_proj", "model.layers.45.mlp.experts.111.gate_proj", "model.layers.45.mlp.experts.112.gate_proj", "model.layers.45.mlp.experts.113.gate_proj", "model.layers.45.mlp.experts.114.gate_proj", "model.layers.45.mlp.experts.115.gate_proj", "model.layers.45.mlp.experts.116.gate_proj", "model.layers.45.mlp.experts.117.gate_proj", "model.layers.45.mlp.experts.118.gate_proj", "model.layers.45.mlp.experts.119.gate_proj", "model.layers.45.mlp.experts.120.gate_proj", "model.layers.45.mlp.experts.121.gate_proj", "model.layers.45.mlp.experts.122.gate_proj", "model.layers.45.mlp.experts.123.gate_proj", "model.layers.45.mlp.experts.124.gate_proj", "model.layers.45.mlp.experts.125.gate_proj", "model.layers.45.mlp.experts.126.gate_proj", "model.layers.45.mlp.experts.127.gate_proj", "model.layers.45.mlp.experts.128.gate_proj", "model.layers.45.mlp.experts.129.gate_proj", "model.layers.45.mlp.experts.130.gate_proj", "model.layers.45.mlp.experts.131.gate_proj", "model.layers.45.mlp.experts.132.gate_proj", "model.layers.45.mlp.experts.133.gate_proj", "model.layers.45.mlp.experts.134.gate_proj", "model.layers.45.mlp.experts.135.gate_proj", "model.layers.45.mlp.experts.136.gate_proj", "model.layers.45.mlp.experts.137.gate_proj", "model.layers.45.mlp.experts.138.gate_proj", "model.layers.45.mlp.experts.139.gate_proj", "model.layers.45.mlp.experts.140.gate_proj", "model.layers.45.mlp.experts.141.gate_proj", "model.layers.45.mlp.experts.142.gate_proj", "model.layers.45.mlp.experts.143.gate_proj", "model.layers.45.mlp.experts.144.gate_proj", "model.layers.45.mlp.experts.145.gate_proj", "model.layers.45.mlp.experts.146.gate_proj", "model.layers.45.mlp.experts.147.gate_proj", "model.layers.45.mlp.experts.148.gate_proj", "model.layers.45.mlp.experts.149.gate_proj", "model.layers.45.mlp.experts.150.gate_proj", "model.layers.45.mlp.experts.151.gate_proj", "model.layers.45.mlp.experts.152.gate_proj", "model.layers.45.mlp.experts.153.gate_proj", "model.layers.45.mlp.experts.154.gate_proj", "model.layers.45.mlp.experts.155.gate_proj", "model.layers.45.mlp.experts.156.gate_proj", "model.layers.45.mlp.experts.157.gate_proj", "model.layers.45.mlp.experts.158.gate_proj", "model.layers.45.mlp.experts.159.gate_proj", "model.layers.45.mlp.experts.0.up_proj", "model.layers.45.mlp.experts.1.up_proj", "model.layers.45.mlp.experts.2.up_proj", "model.layers.45.mlp.experts.3.up_proj", "model.layers.45.mlp.experts.4.up_proj", "model.layers.45.mlp.experts.5.up_proj", "model.layers.45.mlp.experts.6.up_proj", "model.layers.45.mlp.experts.7.up_proj", "model.layers.45.mlp.experts.8.up_proj", "model.layers.45.mlp.experts.9.up_proj", "model.layers.45.mlp.experts.10.up_proj", "model.layers.45.mlp.experts.11.up_proj", "model.layers.45.mlp.experts.12.up_proj", "model.layers.45.mlp.experts.13.up_proj", "model.layers.45.mlp.experts.14.up_proj", "model.layers.45.mlp.experts.15.up_proj", "model.layers.45.mlp.experts.16.up_proj", "model.layers.45.mlp.experts.17.up_proj", "model.layers.45.mlp.experts.18.up_proj", "model.layers.45.mlp.experts.19.up_proj", "model.layers.45.mlp.experts.20.up_proj", "model.layers.45.mlp.experts.21.up_proj", "model.layers.45.mlp.experts.22.up_proj", "model.layers.45.mlp.experts.23.up_proj", "model.layers.45.mlp.experts.24.up_proj", "model.layers.45.mlp.experts.25.up_proj", "model.layers.45.mlp.experts.26.up_proj", "model.layers.45.mlp.experts.27.up_proj", "model.layers.45.mlp.experts.28.up_proj", "model.layers.45.mlp.experts.29.up_proj", "model.layers.45.mlp.experts.30.up_proj", "model.layers.45.mlp.experts.31.up_proj", "model.layers.45.mlp.experts.32.up_proj", "model.layers.45.mlp.experts.33.up_proj", "model.layers.45.mlp.experts.34.up_proj", "model.layers.45.mlp.experts.35.up_proj", "model.layers.45.mlp.experts.36.up_proj", "model.layers.45.mlp.experts.37.up_proj", "model.layers.45.mlp.experts.38.up_proj", "model.layers.45.mlp.experts.39.up_proj", "model.layers.45.mlp.experts.40.up_proj", "model.layers.45.mlp.experts.41.up_proj", "model.layers.45.mlp.experts.42.up_proj", "model.layers.45.mlp.experts.43.up_proj", "model.layers.45.mlp.experts.44.up_proj", "model.layers.45.mlp.experts.45.up_proj", "model.layers.45.mlp.experts.46.up_proj", "model.layers.45.mlp.experts.47.up_proj", "model.layers.45.mlp.experts.48.up_proj", "model.layers.45.mlp.experts.49.up_proj", "model.layers.45.mlp.experts.50.up_proj", "model.layers.45.mlp.experts.51.up_proj", "model.layers.45.mlp.experts.52.up_proj", "model.layers.45.mlp.experts.53.up_proj", "model.layers.45.mlp.experts.54.up_proj", "model.layers.45.mlp.experts.55.up_proj", "model.layers.45.mlp.experts.56.up_proj", "model.layers.45.mlp.experts.57.up_proj", "model.layers.45.mlp.experts.58.up_proj", "model.layers.45.mlp.experts.59.up_proj", "model.layers.45.mlp.experts.60.up_proj", "model.layers.45.mlp.experts.61.up_proj", "model.layers.45.mlp.experts.62.up_proj", "model.layers.45.mlp.experts.63.up_proj", "model.layers.45.mlp.experts.64.up_proj", "model.layers.45.mlp.experts.65.up_proj", "model.layers.45.mlp.experts.66.up_proj", "model.layers.45.mlp.experts.67.up_proj", "model.layers.45.mlp.experts.68.up_proj", "model.layers.45.mlp.experts.69.up_proj", "model.layers.45.mlp.experts.70.up_proj", "model.layers.45.mlp.experts.71.up_proj", "model.layers.45.mlp.experts.72.up_proj", "model.layers.45.mlp.experts.73.up_proj", "model.layers.45.mlp.experts.74.up_proj", "model.layers.45.mlp.experts.75.up_proj", "model.layers.45.mlp.experts.76.up_proj", "model.layers.45.mlp.experts.77.up_proj", "model.layers.45.mlp.experts.78.up_proj", "model.layers.45.mlp.experts.79.up_proj", "model.layers.45.mlp.experts.80.up_proj", "model.layers.45.mlp.experts.81.up_proj", "model.layers.45.mlp.experts.82.up_proj", "model.layers.45.mlp.experts.83.up_proj", "model.layers.45.mlp.experts.84.up_proj", "model.layers.45.mlp.experts.85.up_proj", "model.layers.45.mlp.experts.86.up_proj", "model.layers.45.mlp.experts.87.up_proj", "model.layers.45.mlp.experts.88.up_proj", "model.layers.45.mlp.experts.89.up_proj", "model.layers.45.mlp.experts.90.up_proj", "model.layers.45.mlp.experts.91.up_proj", "model.layers.45.mlp.experts.92.up_proj", "model.layers.45.mlp.experts.93.up_proj", "model.layers.45.mlp.experts.94.up_proj", "model.layers.45.mlp.experts.95.up_proj", "model.layers.45.mlp.experts.96.up_proj", "model.layers.45.mlp.experts.97.up_proj", "model.layers.45.mlp.experts.98.up_proj", "model.layers.45.mlp.experts.99.up_proj", "model.layers.45.mlp.experts.100.up_proj", "model.layers.45.mlp.experts.101.up_proj", "model.layers.45.mlp.experts.102.up_proj", "model.layers.45.mlp.experts.103.up_proj", "model.layers.45.mlp.experts.104.up_proj", "model.layers.45.mlp.experts.105.up_proj", "model.layers.45.mlp.experts.106.up_proj", "model.layers.45.mlp.experts.107.up_proj", "model.layers.45.mlp.experts.108.up_proj", "model.layers.45.mlp.experts.109.up_proj", "model.layers.45.mlp.experts.110.up_proj", "model.layers.45.mlp.experts.111.up_proj", "model.layers.45.mlp.experts.112.up_proj", "model.layers.45.mlp.experts.113.up_proj", "model.layers.45.mlp.experts.114.up_proj", "model.layers.45.mlp.experts.115.up_proj", "model.layers.45.mlp.experts.116.up_proj", "model.layers.45.mlp.experts.117.up_proj", "model.layers.45.mlp.experts.118.up_proj", "model.layers.45.mlp.experts.119.up_proj", "model.layers.45.mlp.experts.120.up_proj", "model.layers.45.mlp.experts.121.up_proj", "model.layers.45.mlp.experts.122.up_proj", "model.layers.45.mlp.experts.123.up_proj", "model.layers.45.mlp.experts.124.up_proj", "model.layers.45.mlp.experts.125.up_proj", "model.layers.45.mlp.experts.126.up_proj", "model.layers.45.mlp.experts.127.up_proj", "model.layers.45.mlp.experts.128.up_proj", "model.layers.45.mlp.experts.129.up_proj", "model.layers.45.mlp.experts.130.up_proj", "model.layers.45.mlp.experts.131.up_proj", "model.layers.45.mlp.experts.132.up_proj", "model.layers.45.mlp.experts.133.up_proj", "model.layers.45.mlp.experts.134.up_proj", "model.layers.45.mlp.experts.135.up_proj", "model.layers.45.mlp.experts.136.up_proj", "model.layers.45.mlp.experts.137.up_proj", "model.layers.45.mlp.experts.138.up_proj", "model.layers.45.mlp.experts.139.up_proj", "model.layers.45.mlp.experts.140.up_proj", "model.layers.45.mlp.experts.141.up_proj", "model.layers.45.mlp.experts.142.up_proj", "model.layers.45.mlp.experts.143.up_proj", "model.layers.45.mlp.experts.144.up_proj", "model.layers.45.mlp.experts.145.up_proj", "model.layers.45.mlp.experts.146.up_proj", "model.layers.45.mlp.experts.147.up_proj", "model.layers.45.mlp.experts.148.up_proj", "model.layers.45.mlp.experts.149.up_proj", "model.layers.45.mlp.experts.150.up_proj", "model.layers.45.mlp.experts.151.up_proj", "model.layers.45.mlp.experts.152.up_proj", "model.layers.45.mlp.experts.153.up_proj", "model.layers.45.mlp.experts.154.up_proj", "model.layers.45.mlp.experts.155.up_proj", "model.layers.45.mlp.experts.156.up_proj", "model.layers.45.mlp.experts.157.up_proj", "model.layers.45.mlp.experts.158.up_proj", "model.layers.45.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.783150946721478e-06, "dbits": 5033164800 } ] }, { "idx": 272, "layers": [ "model.layers.45.mlp.experts.0.down_proj", "model.layers.45.mlp.experts.1.down_proj", "model.layers.45.mlp.experts.2.down_proj", "model.layers.45.mlp.experts.3.down_proj", "model.layers.45.mlp.experts.4.down_proj", "model.layers.45.mlp.experts.5.down_proj", "model.layers.45.mlp.experts.6.down_proj", "model.layers.45.mlp.experts.7.down_proj", "model.layers.45.mlp.experts.8.down_proj", "model.layers.45.mlp.experts.9.down_proj", "model.layers.45.mlp.experts.10.down_proj", "model.layers.45.mlp.experts.11.down_proj", "model.layers.45.mlp.experts.12.down_proj", "model.layers.45.mlp.experts.13.down_proj", "model.layers.45.mlp.experts.14.down_proj", "model.layers.45.mlp.experts.15.down_proj", "model.layers.45.mlp.experts.16.down_proj", "model.layers.45.mlp.experts.17.down_proj", "model.layers.45.mlp.experts.18.down_proj", "model.layers.45.mlp.experts.19.down_proj", "model.layers.45.mlp.experts.20.down_proj", "model.layers.45.mlp.experts.21.down_proj", "model.layers.45.mlp.experts.22.down_proj", "model.layers.45.mlp.experts.23.down_proj", "model.layers.45.mlp.experts.24.down_proj", "model.layers.45.mlp.experts.25.down_proj", "model.layers.45.mlp.experts.26.down_proj", "model.layers.45.mlp.experts.27.down_proj", "model.layers.45.mlp.experts.28.down_proj", "model.layers.45.mlp.experts.29.down_proj", "model.layers.45.mlp.experts.30.down_proj", "model.layers.45.mlp.experts.31.down_proj", "model.layers.45.mlp.experts.32.down_proj", "model.layers.45.mlp.experts.33.down_proj", "model.layers.45.mlp.experts.34.down_proj", "model.layers.45.mlp.experts.35.down_proj", "model.layers.45.mlp.experts.36.down_proj", "model.layers.45.mlp.experts.37.down_proj", "model.layers.45.mlp.experts.38.down_proj", "model.layers.45.mlp.experts.39.down_proj", "model.layers.45.mlp.experts.40.down_proj", "model.layers.45.mlp.experts.41.down_proj", "model.layers.45.mlp.experts.42.down_proj", "model.layers.45.mlp.experts.43.down_proj", "model.layers.45.mlp.experts.44.down_proj", "model.layers.45.mlp.experts.45.down_proj", "model.layers.45.mlp.experts.46.down_proj", "model.layers.45.mlp.experts.47.down_proj", "model.layers.45.mlp.experts.48.down_proj", "model.layers.45.mlp.experts.49.down_proj", "model.layers.45.mlp.experts.50.down_proj", "model.layers.45.mlp.experts.51.down_proj", "model.layers.45.mlp.experts.52.down_proj", "model.layers.45.mlp.experts.53.down_proj", "model.layers.45.mlp.experts.54.down_proj", "model.layers.45.mlp.experts.55.down_proj", "model.layers.45.mlp.experts.56.down_proj", "model.layers.45.mlp.experts.57.down_proj", "model.layers.45.mlp.experts.58.down_proj", "model.layers.45.mlp.experts.59.down_proj", "model.layers.45.mlp.experts.60.down_proj", "model.layers.45.mlp.experts.61.down_proj", "model.layers.45.mlp.experts.62.down_proj", "model.layers.45.mlp.experts.63.down_proj", "model.layers.45.mlp.experts.64.down_proj", "model.layers.45.mlp.experts.65.down_proj", "model.layers.45.mlp.experts.66.down_proj", "model.layers.45.mlp.experts.67.down_proj", "model.layers.45.mlp.experts.68.down_proj", "model.layers.45.mlp.experts.69.down_proj", "model.layers.45.mlp.experts.70.down_proj", "model.layers.45.mlp.experts.71.down_proj", "model.layers.45.mlp.experts.72.down_proj", "model.layers.45.mlp.experts.73.down_proj", "model.layers.45.mlp.experts.74.down_proj", "model.layers.45.mlp.experts.75.down_proj", "model.layers.45.mlp.experts.76.down_proj", "model.layers.45.mlp.experts.77.down_proj", "model.layers.45.mlp.experts.78.down_proj", "model.layers.45.mlp.experts.79.down_proj", "model.layers.45.mlp.experts.80.down_proj", "model.layers.45.mlp.experts.81.down_proj", "model.layers.45.mlp.experts.82.down_proj", "model.layers.45.mlp.experts.83.down_proj", "model.layers.45.mlp.experts.84.down_proj", "model.layers.45.mlp.experts.85.down_proj", "model.layers.45.mlp.experts.86.down_proj", "model.layers.45.mlp.experts.87.down_proj", "model.layers.45.mlp.experts.88.down_proj", "model.layers.45.mlp.experts.89.down_proj", "model.layers.45.mlp.experts.90.down_proj", "model.layers.45.mlp.experts.91.down_proj", "model.layers.45.mlp.experts.92.down_proj", "model.layers.45.mlp.experts.93.down_proj", "model.layers.45.mlp.experts.94.down_proj", "model.layers.45.mlp.experts.95.down_proj", "model.layers.45.mlp.experts.96.down_proj", "model.layers.45.mlp.experts.97.down_proj", "model.layers.45.mlp.experts.98.down_proj", "model.layers.45.mlp.experts.99.down_proj", "model.layers.45.mlp.experts.100.down_proj", "model.layers.45.mlp.experts.101.down_proj", "model.layers.45.mlp.experts.102.down_proj", "model.layers.45.mlp.experts.103.down_proj", "model.layers.45.mlp.experts.104.down_proj", "model.layers.45.mlp.experts.105.down_proj", "model.layers.45.mlp.experts.106.down_proj", "model.layers.45.mlp.experts.107.down_proj", "model.layers.45.mlp.experts.108.down_proj", "model.layers.45.mlp.experts.109.down_proj", "model.layers.45.mlp.experts.110.down_proj", "model.layers.45.mlp.experts.111.down_proj", "model.layers.45.mlp.experts.112.down_proj", "model.layers.45.mlp.experts.113.down_proj", "model.layers.45.mlp.experts.114.down_proj", "model.layers.45.mlp.experts.115.down_proj", "model.layers.45.mlp.experts.116.down_proj", "model.layers.45.mlp.experts.117.down_proj", "model.layers.45.mlp.experts.118.down_proj", "model.layers.45.mlp.experts.119.down_proj", "model.layers.45.mlp.experts.120.down_proj", "model.layers.45.mlp.experts.121.down_proj", "model.layers.45.mlp.experts.122.down_proj", "model.layers.45.mlp.experts.123.down_proj", "model.layers.45.mlp.experts.124.down_proj", "model.layers.45.mlp.experts.125.down_proj", "model.layers.45.mlp.experts.126.down_proj", "model.layers.45.mlp.experts.127.down_proj", "model.layers.45.mlp.experts.128.down_proj", "model.layers.45.mlp.experts.129.down_proj", "model.layers.45.mlp.experts.130.down_proj", "model.layers.45.mlp.experts.131.down_proj", "model.layers.45.mlp.experts.132.down_proj", "model.layers.45.mlp.experts.133.down_proj", "model.layers.45.mlp.experts.134.down_proj", "model.layers.45.mlp.experts.135.down_proj", "model.layers.45.mlp.experts.136.down_proj", "model.layers.45.mlp.experts.137.down_proj", "model.layers.45.mlp.experts.138.down_proj", "model.layers.45.mlp.experts.139.down_proj", "model.layers.45.mlp.experts.140.down_proj", "model.layers.45.mlp.experts.141.down_proj", "model.layers.45.mlp.experts.142.down_proj", "model.layers.45.mlp.experts.143.down_proj", "model.layers.45.mlp.experts.144.down_proj", "model.layers.45.mlp.experts.145.down_proj", "model.layers.45.mlp.experts.146.down_proj", "model.layers.45.mlp.experts.147.down_proj", "model.layers.45.mlp.experts.148.down_proj", "model.layers.45.mlp.experts.149.down_proj", "model.layers.45.mlp.experts.150.down_proj", "model.layers.45.mlp.experts.151.down_proj", "model.layers.45.mlp.experts.152.down_proj", "model.layers.45.mlp.experts.153.down_proj", "model.layers.45.mlp.experts.154.down_proj", "model.layers.45.mlp.experts.155.down_proj", "model.layers.45.mlp.experts.156.down_proj", "model.layers.45.mlp.experts.157.down_proj", "model.layers.45.mlp.experts.158.down_proj", "model.layers.45.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.452094875276132e-06, "dbits": 2516582400 } ] }, { "idx": 273, "layers": [ "model.layers.46.self_attn.q_proj" ], "candidates": [ { "dkld": 4.402044578455432e-06, "dbits": 125829120 } ] }, { "idx": 274, "layers": [ "model.layers.46.self_attn.k_proj", "model.layers.46.self_attn.v_proj" ], "candidates": [ { "dkld": 1.956414780579464e-06, "dbits": 20971520 } ] }, { "idx": 275, "layers": [ "model.layers.46.self_attn.o_proj" ], "candidates": [ { "dkld": 4.4504588004202084e-06, "dbits": 125829120 } ] }, { "idx": 276, "layers": [ "model.layers.46.mlp.shared_experts.gate_proj", "model.layers.46.mlp.shared_experts.up_proj", "model.layers.46.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.0911073079332276e-06, "dbits": 47185920 } ] }, { "idx": 277, "layers": [ "model.layers.46.mlp.experts.0.gate_proj", "model.layers.46.mlp.experts.1.gate_proj", "model.layers.46.mlp.experts.2.gate_proj", "model.layers.46.mlp.experts.3.gate_proj", "model.layers.46.mlp.experts.4.gate_proj", "model.layers.46.mlp.experts.5.gate_proj", "model.layers.46.mlp.experts.6.gate_proj", "model.layers.46.mlp.experts.7.gate_proj", "model.layers.46.mlp.experts.8.gate_proj", "model.layers.46.mlp.experts.9.gate_proj", "model.layers.46.mlp.experts.10.gate_proj", "model.layers.46.mlp.experts.11.gate_proj", "model.layers.46.mlp.experts.12.gate_proj", "model.layers.46.mlp.experts.13.gate_proj", "model.layers.46.mlp.experts.14.gate_proj", "model.layers.46.mlp.experts.15.gate_proj", "model.layers.46.mlp.experts.16.gate_proj", "model.layers.46.mlp.experts.17.gate_proj", "model.layers.46.mlp.experts.18.gate_proj", "model.layers.46.mlp.experts.19.gate_proj", "model.layers.46.mlp.experts.20.gate_proj", "model.layers.46.mlp.experts.21.gate_proj", "model.layers.46.mlp.experts.22.gate_proj", "model.layers.46.mlp.experts.23.gate_proj", "model.layers.46.mlp.experts.24.gate_proj", "model.layers.46.mlp.experts.25.gate_proj", "model.layers.46.mlp.experts.26.gate_proj", "model.layers.46.mlp.experts.27.gate_proj", "model.layers.46.mlp.experts.28.gate_proj", "model.layers.46.mlp.experts.29.gate_proj", "model.layers.46.mlp.experts.30.gate_proj", "model.layers.46.mlp.experts.31.gate_proj", "model.layers.46.mlp.experts.32.gate_proj", "model.layers.46.mlp.experts.33.gate_proj", "model.layers.46.mlp.experts.34.gate_proj", "model.layers.46.mlp.experts.35.gate_proj", "model.layers.46.mlp.experts.36.gate_proj", "model.layers.46.mlp.experts.37.gate_proj", "model.layers.46.mlp.experts.38.gate_proj", "model.layers.46.mlp.experts.39.gate_proj", "model.layers.46.mlp.experts.40.gate_proj", "model.layers.46.mlp.experts.41.gate_proj", "model.layers.46.mlp.experts.42.gate_proj", "model.layers.46.mlp.experts.43.gate_proj", "model.layers.46.mlp.experts.44.gate_proj", "model.layers.46.mlp.experts.45.gate_proj", "model.layers.46.mlp.experts.46.gate_proj", "model.layers.46.mlp.experts.47.gate_proj", "model.layers.46.mlp.experts.48.gate_proj", "model.layers.46.mlp.experts.49.gate_proj", "model.layers.46.mlp.experts.50.gate_proj", "model.layers.46.mlp.experts.51.gate_proj", "model.layers.46.mlp.experts.52.gate_proj", "model.layers.46.mlp.experts.53.gate_proj", "model.layers.46.mlp.experts.54.gate_proj", "model.layers.46.mlp.experts.55.gate_proj", "model.layers.46.mlp.experts.56.gate_proj", "model.layers.46.mlp.experts.57.gate_proj", "model.layers.46.mlp.experts.58.gate_proj", "model.layers.46.mlp.experts.59.gate_proj", "model.layers.46.mlp.experts.60.gate_proj", "model.layers.46.mlp.experts.61.gate_proj", "model.layers.46.mlp.experts.62.gate_proj", "model.layers.46.mlp.experts.63.gate_proj", "model.layers.46.mlp.experts.64.gate_proj", "model.layers.46.mlp.experts.65.gate_proj", "model.layers.46.mlp.experts.66.gate_proj", "model.layers.46.mlp.experts.67.gate_proj", "model.layers.46.mlp.experts.68.gate_proj", "model.layers.46.mlp.experts.69.gate_proj", "model.layers.46.mlp.experts.70.gate_proj", "model.layers.46.mlp.experts.71.gate_proj", "model.layers.46.mlp.experts.72.gate_proj", "model.layers.46.mlp.experts.73.gate_proj", "model.layers.46.mlp.experts.74.gate_proj", "model.layers.46.mlp.experts.75.gate_proj", "model.layers.46.mlp.experts.76.gate_proj", "model.layers.46.mlp.experts.77.gate_proj", "model.layers.46.mlp.experts.78.gate_proj", "model.layers.46.mlp.experts.79.gate_proj", "model.layers.46.mlp.experts.80.gate_proj", "model.layers.46.mlp.experts.81.gate_proj", "model.layers.46.mlp.experts.82.gate_proj", "model.layers.46.mlp.experts.83.gate_proj", "model.layers.46.mlp.experts.84.gate_proj", "model.layers.46.mlp.experts.85.gate_proj", "model.layers.46.mlp.experts.86.gate_proj", "model.layers.46.mlp.experts.87.gate_proj", "model.layers.46.mlp.experts.88.gate_proj", "model.layers.46.mlp.experts.89.gate_proj", "model.layers.46.mlp.experts.90.gate_proj", "model.layers.46.mlp.experts.91.gate_proj", "model.layers.46.mlp.experts.92.gate_proj", "model.layers.46.mlp.experts.93.gate_proj", "model.layers.46.mlp.experts.94.gate_proj", "model.layers.46.mlp.experts.95.gate_proj", "model.layers.46.mlp.experts.96.gate_proj", "model.layers.46.mlp.experts.97.gate_proj", "model.layers.46.mlp.experts.98.gate_proj", "model.layers.46.mlp.experts.99.gate_proj", "model.layers.46.mlp.experts.100.gate_proj", "model.layers.46.mlp.experts.101.gate_proj", "model.layers.46.mlp.experts.102.gate_proj", "model.layers.46.mlp.experts.103.gate_proj", "model.layers.46.mlp.experts.104.gate_proj", "model.layers.46.mlp.experts.105.gate_proj", "model.layers.46.mlp.experts.106.gate_proj", "model.layers.46.mlp.experts.107.gate_proj", "model.layers.46.mlp.experts.108.gate_proj", "model.layers.46.mlp.experts.109.gate_proj", "model.layers.46.mlp.experts.110.gate_proj", "model.layers.46.mlp.experts.111.gate_proj", "model.layers.46.mlp.experts.112.gate_proj", "model.layers.46.mlp.experts.113.gate_proj", "model.layers.46.mlp.experts.114.gate_proj", "model.layers.46.mlp.experts.115.gate_proj", "model.layers.46.mlp.experts.116.gate_proj", "model.layers.46.mlp.experts.117.gate_proj", "model.layers.46.mlp.experts.118.gate_proj", "model.layers.46.mlp.experts.119.gate_proj", "model.layers.46.mlp.experts.120.gate_proj", "model.layers.46.mlp.experts.121.gate_proj", "model.layers.46.mlp.experts.122.gate_proj", "model.layers.46.mlp.experts.123.gate_proj", "model.layers.46.mlp.experts.124.gate_proj", "model.layers.46.mlp.experts.125.gate_proj", "model.layers.46.mlp.experts.126.gate_proj", "model.layers.46.mlp.experts.127.gate_proj", "model.layers.46.mlp.experts.128.gate_proj", "model.layers.46.mlp.experts.129.gate_proj", "model.layers.46.mlp.experts.130.gate_proj", "model.layers.46.mlp.experts.131.gate_proj", "model.layers.46.mlp.experts.132.gate_proj", "model.layers.46.mlp.experts.133.gate_proj", "model.layers.46.mlp.experts.134.gate_proj", "model.layers.46.mlp.experts.135.gate_proj", "model.layers.46.mlp.experts.136.gate_proj", "model.layers.46.mlp.experts.137.gate_proj", "model.layers.46.mlp.experts.138.gate_proj", "model.layers.46.mlp.experts.139.gate_proj", "model.layers.46.mlp.experts.140.gate_proj", "model.layers.46.mlp.experts.141.gate_proj", "model.layers.46.mlp.experts.142.gate_proj", "model.layers.46.mlp.experts.143.gate_proj", "model.layers.46.mlp.experts.144.gate_proj", "model.layers.46.mlp.experts.145.gate_proj", "model.layers.46.mlp.experts.146.gate_proj", "model.layers.46.mlp.experts.147.gate_proj", "model.layers.46.mlp.experts.148.gate_proj", "model.layers.46.mlp.experts.149.gate_proj", "model.layers.46.mlp.experts.150.gate_proj", "model.layers.46.mlp.experts.151.gate_proj", "model.layers.46.mlp.experts.152.gate_proj", "model.layers.46.mlp.experts.153.gate_proj", "model.layers.46.mlp.experts.154.gate_proj", "model.layers.46.mlp.experts.155.gate_proj", "model.layers.46.mlp.experts.156.gate_proj", "model.layers.46.mlp.experts.157.gate_proj", "model.layers.46.mlp.experts.158.gate_proj", "model.layers.46.mlp.experts.159.gate_proj", "model.layers.46.mlp.experts.0.up_proj", "model.layers.46.mlp.experts.1.up_proj", "model.layers.46.mlp.experts.2.up_proj", "model.layers.46.mlp.experts.3.up_proj", "model.layers.46.mlp.experts.4.up_proj", "model.layers.46.mlp.experts.5.up_proj", "model.layers.46.mlp.experts.6.up_proj", "model.layers.46.mlp.experts.7.up_proj", "model.layers.46.mlp.experts.8.up_proj", "model.layers.46.mlp.experts.9.up_proj", "model.layers.46.mlp.experts.10.up_proj", "model.layers.46.mlp.experts.11.up_proj", "model.layers.46.mlp.experts.12.up_proj", "model.layers.46.mlp.experts.13.up_proj", "model.layers.46.mlp.experts.14.up_proj", "model.layers.46.mlp.experts.15.up_proj", "model.layers.46.mlp.experts.16.up_proj", "model.layers.46.mlp.experts.17.up_proj", "model.layers.46.mlp.experts.18.up_proj", "model.layers.46.mlp.experts.19.up_proj", "model.layers.46.mlp.experts.20.up_proj", "model.layers.46.mlp.experts.21.up_proj", "model.layers.46.mlp.experts.22.up_proj", "model.layers.46.mlp.experts.23.up_proj", "model.layers.46.mlp.experts.24.up_proj", "model.layers.46.mlp.experts.25.up_proj", "model.layers.46.mlp.experts.26.up_proj", "model.layers.46.mlp.experts.27.up_proj", "model.layers.46.mlp.experts.28.up_proj", "model.layers.46.mlp.experts.29.up_proj", "model.layers.46.mlp.experts.30.up_proj", "model.layers.46.mlp.experts.31.up_proj", "model.layers.46.mlp.experts.32.up_proj", "model.layers.46.mlp.experts.33.up_proj", "model.layers.46.mlp.experts.34.up_proj", "model.layers.46.mlp.experts.35.up_proj", "model.layers.46.mlp.experts.36.up_proj", "model.layers.46.mlp.experts.37.up_proj", "model.layers.46.mlp.experts.38.up_proj", "model.layers.46.mlp.experts.39.up_proj", "model.layers.46.mlp.experts.40.up_proj", "model.layers.46.mlp.experts.41.up_proj", "model.layers.46.mlp.experts.42.up_proj", "model.layers.46.mlp.experts.43.up_proj", "model.layers.46.mlp.experts.44.up_proj", "model.layers.46.mlp.experts.45.up_proj", "model.layers.46.mlp.experts.46.up_proj", "model.layers.46.mlp.experts.47.up_proj", "model.layers.46.mlp.experts.48.up_proj", "model.layers.46.mlp.experts.49.up_proj", "model.layers.46.mlp.experts.50.up_proj", "model.layers.46.mlp.experts.51.up_proj", "model.layers.46.mlp.experts.52.up_proj", "model.layers.46.mlp.experts.53.up_proj", "model.layers.46.mlp.experts.54.up_proj", "model.layers.46.mlp.experts.55.up_proj", "model.layers.46.mlp.experts.56.up_proj", "model.layers.46.mlp.experts.57.up_proj", "model.layers.46.mlp.experts.58.up_proj", "model.layers.46.mlp.experts.59.up_proj", "model.layers.46.mlp.experts.60.up_proj", "model.layers.46.mlp.experts.61.up_proj", "model.layers.46.mlp.experts.62.up_proj", "model.layers.46.mlp.experts.63.up_proj", "model.layers.46.mlp.experts.64.up_proj", "model.layers.46.mlp.experts.65.up_proj", "model.layers.46.mlp.experts.66.up_proj", "model.layers.46.mlp.experts.67.up_proj", "model.layers.46.mlp.experts.68.up_proj", "model.layers.46.mlp.experts.69.up_proj", "model.layers.46.mlp.experts.70.up_proj", "model.layers.46.mlp.experts.71.up_proj", "model.layers.46.mlp.experts.72.up_proj", "model.layers.46.mlp.experts.73.up_proj", "model.layers.46.mlp.experts.74.up_proj", "model.layers.46.mlp.experts.75.up_proj", "model.layers.46.mlp.experts.76.up_proj", "model.layers.46.mlp.experts.77.up_proj", "model.layers.46.mlp.experts.78.up_proj", "model.layers.46.mlp.experts.79.up_proj", "model.layers.46.mlp.experts.80.up_proj", "model.layers.46.mlp.experts.81.up_proj", "model.layers.46.mlp.experts.82.up_proj", "model.layers.46.mlp.experts.83.up_proj", "model.layers.46.mlp.experts.84.up_proj", "model.layers.46.mlp.experts.85.up_proj", "model.layers.46.mlp.experts.86.up_proj", "model.layers.46.mlp.experts.87.up_proj", "model.layers.46.mlp.experts.88.up_proj", "model.layers.46.mlp.experts.89.up_proj", "model.layers.46.mlp.experts.90.up_proj", "model.layers.46.mlp.experts.91.up_proj", "model.layers.46.mlp.experts.92.up_proj", "model.layers.46.mlp.experts.93.up_proj", "model.layers.46.mlp.experts.94.up_proj", "model.layers.46.mlp.experts.95.up_proj", "model.layers.46.mlp.experts.96.up_proj", "model.layers.46.mlp.experts.97.up_proj", "model.layers.46.mlp.experts.98.up_proj", "model.layers.46.mlp.experts.99.up_proj", "model.layers.46.mlp.experts.100.up_proj", "model.layers.46.mlp.experts.101.up_proj", "model.layers.46.mlp.experts.102.up_proj", "model.layers.46.mlp.experts.103.up_proj", "model.layers.46.mlp.experts.104.up_proj", "model.layers.46.mlp.experts.105.up_proj", "model.layers.46.mlp.experts.106.up_proj", "model.layers.46.mlp.experts.107.up_proj", "model.layers.46.mlp.experts.108.up_proj", "model.layers.46.mlp.experts.109.up_proj", "model.layers.46.mlp.experts.110.up_proj", "model.layers.46.mlp.experts.111.up_proj", "model.layers.46.mlp.experts.112.up_proj", "model.layers.46.mlp.experts.113.up_proj", "model.layers.46.mlp.experts.114.up_proj", "model.layers.46.mlp.experts.115.up_proj", "model.layers.46.mlp.experts.116.up_proj", "model.layers.46.mlp.experts.117.up_proj", "model.layers.46.mlp.experts.118.up_proj", "model.layers.46.mlp.experts.119.up_proj", "model.layers.46.mlp.experts.120.up_proj", "model.layers.46.mlp.experts.121.up_proj", "model.layers.46.mlp.experts.122.up_proj", "model.layers.46.mlp.experts.123.up_proj", "model.layers.46.mlp.experts.124.up_proj", "model.layers.46.mlp.experts.125.up_proj", "model.layers.46.mlp.experts.126.up_proj", "model.layers.46.mlp.experts.127.up_proj", "model.layers.46.mlp.experts.128.up_proj", "model.layers.46.mlp.experts.129.up_proj", "model.layers.46.mlp.experts.130.up_proj", "model.layers.46.mlp.experts.131.up_proj", "model.layers.46.mlp.experts.132.up_proj", "model.layers.46.mlp.experts.133.up_proj", "model.layers.46.mlp.experts.134.up_proj", "model.layers.46.mlp.experts.135.up_proj", "model.layers.46.mlp.experts.136.up_proj", "model.layers.46.mlp.experts.137.up_proj", "model.layers.46.mlp.experts.138.up_proj", "model.layers.46.mlp.experts.139.up_proj", "model.layers.46.mlp.experts.140.up_proj", "model.layers.46.mlp.experts.141.up_proj", "model.layers.46.mlp.experts.142.up_proj", "model.layers.46.mlp.experts.143.up_proj", "model.layers.46.mlp.experts.144.up_proj", "model.layers.46.mlp.experts.145.up_proj", "model.layers.46.mlp.experts.146.up_proj", "model.layers.46.mlp.experts.147.up_proj", "model.layers.46.mlp.experts.148.up_proj", "model.layers.46.mlp.experts.149.up_proj", "model.layers.46.mlp.experts.150.up_proj", "model.layers.46.mlp.experts.151.up_proj", "model.layers.46.mlp.experts.152.up_proj", "model.layers.46.mlp.experts.153.up_proj", "model.layers.46.mlp.experts.154.up_proj", "model.layers.46.mlp.experts.155.up_proj", "model.layers.46.mlp.experts.156.up_proj", "model.layers.46.mlp.experts.157.up_proj", "model.layers.46.mlp.experts.158.up_proj", "model.layers.46.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.154183600097851e-06, "dbits": 5033164800 } ] }, { "idx": 278, "layers": [ "model.layers.46.mlp.experts.0.down_proj", "model.layers.46.mlp.experts.1.down_proj", "model.layers.46.mlp.experts.2.down_proj", "model.layers.46.mlp.experts.3.down_proj", "model.layers.46.mlp.experts.4.down_proj", "model.layers.46.mlp.experts.5.down_proj", "model.layers.46.mlp.experts.6.down_proj", "model.layers.46.mlp.experts.7.down_proj", "model.layers.46.mlp.experts.8.down_proj", "model.layers.46.mlp.experts.9.down_proj", "model.layers.46.mlp.experts.10.down_proj", "model.layers.46.mlp.experts.11.down_proj", "model.layers.46.mlp.experts.12.down_proj", "model.layers.46.mlp.experts.13.down_proj", "model.layers.46.mlp.experts.14.down_proj", "model.layers.46.mlp.experts.15.down_proj", "model.layers.46.mlp.experts.16.down_proj", "model.layers.46.mlp.experts.17.down_proj", "model.layers.46.mlp.experts.18.down_proj", "model.layers.46.mlp.experts.19.down_proj", "model.layers.46.mlp.experts.20.down_proj", "model.layers.46.mlp.experts.21.down_proj", "model.layers.46.mlp.experts.22.down_proj", "model.layers.46.mlp.experts.23.down_proj", "model.layers.46.mlp.experts.24.down_proj", "model.layers.46.mlp.experts.25.down_proj", "model.layers.46.mlp.experts.26.down_proj", "model.layers.46.mlp.experts.27.down_proj", "model.layers.46.mlp.experts.28.down_proj", "model.layers.46.mlp.experts.29.down_proj", "model.layers.46.mlp.experts.30.down_proj", "model.layers.46.mlp.experts.31.down_proj", "model.layers.46.mlp.experts.32.down_proj", "model.layers.46.mlp.experts.33.down_proj", "model.layers.46.mlp.experts.34.down_proj", "model.layers.46.mlp.experts.35.down_proj", "model.layers.46.mlp.experts.36.down_proj", "model.layers.46.mlp.experts.37.down_proj", "model.layers.46.mlp.experts.38.down_proj", "model.layers.46.mlp.experts.39.down_proj", "model.layers.46.mlp.experts.40.down_proj", "model.layers.46.mlp.experts.41.down_proj", "model.layers.46.mlp.experts.42.down_proj", "model.layers.46.mlp.experts.43.down_proj", "model.layers.46.mlp.experts.44.down_proj", "model.layers.46.mlp.experts.45.down_proj", "model.layers.46.mlp.experts.46.down_proj", "model.layers.46.mlp.experts.47.down_proj", "model.layers.46.mlp.experts.48.down_proj", "model.layers.46.mlp.experts.49.down_proj", "model.layers.46.mlp.experts.50.down_proj", "model.layers.46.mlp.experts.51.down_proj", "model.layers.46.mlp.experts.52.down_proj", "model.layers.46.mlp.experts.53.down_proj", "model.layers.46.mlp.experts.54.down_proj", "model.layers.46.mlp.experts.55.down_proj", "model.layers.46.mlp.experts.56.down_proj", "model.layers.46.mlp.experts.57.down_proj", "model.layers.46.mlp.experts.58.down_proj", "model.layers.46.mlp.experts.59.down_proj", "model.layers.46.mlp.experts.60.down_proj", "model.layers.46.mlp.experts.61.down_proj", "model.layers.46.mlp.experts.62.down_proj", "model.layers.46.mlp.experts.63.down_proj", "model.layers.46.mlp.experts.64.down_proj", "model.layers.46.mlp.experts.65.down_proj", "model.layers.46.mlp.experts.66.down_proj", "model.layers.46.mlp.experts.67.down_proj", "model.layers.46.mlp.experts.68.down_proj", "model.layers.46.mlp.experts.69.down_proj", "model.layers.46.mlp.experts.70.down_proj", "model.layers.46.mlp.experts.71.down_proj", "model.layers.46.mlp.experts.72.down_proj", "model.layers.46.mlp.experts.73.down_proj", "model.layers.46.mlp.experts.74.down_proj", "model.layers.46.mlp.experts.75.down_proj", "model.layers.46.mlp.experts.76.down_proj", "model.layers.46.mlp.experts.77.down_proj", "model.layers.46.mlp.experts.78.down_proj", "model.layers.46.mlp.experts.79.down_proj", "model.layers.46.mlp.experts.80.down_proj", "model.layers.46.mlp.experts.81.down_proj", "model.layers.46.mlp.experts.82.down_proj", "model.layers.46.mlp.experts.83.down_proj", "model.layers.46.mlp.experts.84.down_proj", "model.layers.46.mlp.experts.85.down_proj", "model.layers.46.mlp.experts.86.down_proj", "model.layers.46.mlp.experts.87.down_proj", "model.layers.46.mlp.experts.88.down_proj", "model.layers.46.mlp.experts.89.down_proj", "model.layers.46.mlp.experts.90.down_proj", "model.layers.46.mlp.experts.91.down_proj", "model.layers.46.mlp.experts.92.down_proj", "model.layers.46.mlp.experts.93.down_proj", "model.layers.46.mlp.experts.94.down_proj", "model.layers.46.mlp.experts.95.down_proj", "model.layers.46.mlp.experts.96.down_proj", "model.layers.46.mlp.experts.97.down_proj", "model.layers.46.mlp.experts.98.down_proj", "model.layers.46.mlp.experts.99.down_proj", "model.layers.46.mlp.experts.100.down_proj", "model.layers.46.mlp.experts.101.down_proj", "model.layers.46.mlp.experts.102.down_proj", "model.layers.46.mlp.experts.103.down_proj", "model.layers.46.mlp.experts.104.down_proj", "model.layers.46.mlp.experts.105.down_proj", "model.layers.46.mlp.experts.106.down_proj", "model.layers.46.mlp.experts.107.down_proj", "model.layers.46.mlp.experts.108.down_proj", "model.layers.46.mlp.experts.109.down_proj", "model.layers.46.mlp.experts.110.down_proj", "model.layers.46.mlp.experts.111.down_proj", "model.layers.46.mlp.experts.112.down_proj", "model.layers.46.mlp.experts.113.down_proj", "model.layers.46.mlp.experts.114.down_proj", "model.layers.46.mlp.experts.115.down_proj", "model.layers.46.mlp.experts.116.down_proj", "model.layers.46.mlp.experts.117.down_proj", "model.layers.46.mlp.experts.118.down_proj", "model.layers.46.mlp.experts.119.down_proj", "model.layers.46.mlp.experts.120.down_proj", "model.layers.46.mlp.experts.121.down_proj", "model.layers.46.mlp.experts.122.down_proj", "model.layers.46.mlp.experts.123.down_proj", "model.layers.46.mlp.experts.124.down_proj", "model.layers.46.mlp.experts.125.down_proj", "model.layers.46.mlp.experts.126.down_proj", "model.layers.46.mlp.experts.127.down_proj", "model.layers.46.mlp.experts.128.down_proj", "model.layers.46.mlp.experts.129.down_proj", "model.layers.46.mlp.experts.130.down_proj", "model.layers.46.mlp.experts.131.down_proj", "model.layers.46.mlp.experts.132.down_proj", "model.layers.46.mlp.experts.133.down_proj", "model.layers.46.mlp.experts.134.down_proj", "model.layers.46.mlp.experts.135.down_proj", "model.layers.46.mlp.experts.136.down_proj", "model.layers.46.mlp.experts.137.down_proj", "model.layers.46.mlp.experts.138.down_proj", "model.layers.46.mlp.experts.139.down_proj", "model.layers.46.mlp.experts.140.down_proj", "model.layers.46.mlp.experts.141.down_proj", "model.layers.46.mlp.experts.142.down_proj", "model.layers.46.mlp.experts.143.down_proj", "model.layers.46.mlp.experts.144.down_proj", "model.layers.46.mlp.experts.145.down_proj", "model.layers.46.mlp.experts.146.down_proj", "model.layers.46.mlp.experts.147.down_proj", "model.layers.46.mlp.experts.148.down_proj", "model.layers.46.mlp.experts.149.down_proj", "model.layers.46.mlp.experts.150.down_proj", "model.layers.46.mlp.experts.151.down_proj", "model.layers.46.mlp.experts.152.down_proj", "model.layers.46.mlp.experts.153.down_proj", "model.layers.46.mlp.experts.154.down_proj", "model.layers.46.mlp.experts.155.down_proj", "model.layers.46.mlp.experts.156.down_proj", "model.layers.46.mlp.experts.157.down_proj", "model.layers.46.mlp.experts.158.down_proj", "model.layers.46.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.1444033589213653e-06, "dbits": 2516582400 } ] }, { "idx": 279, "layers": [ "model.layers.47.self_attn.q_proj" ], "candidates": [ { "dkld": 2.1343235857784965e-06, "dbits": 125829120 } ] }, { "idx": 280, "layers": [ "model.layers.47.self_attn.k_proj", "model.layers.47.self_attn.v_proj" ], "candidates": [ { "dkld": 7.7848817454651e-06, "dbits": 20971520 } ] }, { "idx": 281, "layers": [ "model.layers.47.self_attn.o_proj" ], "candidates": [ { "dkld": 1.5416298992931843e-06, "dbits": 125829120 } ] }, { "idx": 282, "layers": [ "model.layers.47.mlp.shared_experts.gate_proj", "model.layers.47.mlp.shared_experts.up_proj", "model.layers.47.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.8253940399736383e-06, "dbits": 47185920 } ] }, { "idx": 283, "layers": [ "model.layers.47.mlp.experts.0.gate_proj", "model.layers.47.mlp.experts.1.gate_proj", "model.layers.47.mlp.experts.2.gate_proj", "model.layers.47.mlp.experts.3.gate_proj", "model.layers.47.mlp.experts.4.gate_proj", "model.layers.47.mlp.experts.5.gate_proj", "model.layers.47.mlp.experts.6.gate_proj", "model.layers.47.mlp.experts.7.gate_proj", "model.layers.47.mlp.experts.8.gate_proj", "model.layers.47.mlp.experts.9.gate_proj", "model.layers.47.mlp.experts.10.gate_proj", "model.layers.47.mlp.experts.11.gate_proj", "model.layers.47.mlp.experts.12.gate_proj", "model.layers.47.mlp.experts.13.gate_proj", "model.layers.47.mlp.experts.14.gate_proj", "model.layers.47.mlp.experts.15.gate_proj", "model.layers.47.mlp.experts.16.gate_proj", "model.layers.47.mlp.experts.17.gate_proj", "model.layers.47.mlp.experts.18.gate_proj", "model.layers.47.mlp.experts.19.gate_proj", "model.layers.47.mlp.experts.20.gate_proj", "model.layers.47.mlp.experts.21.gate_proj", "model.layers.47.mlp.experts.22.gate_proj", "model.layers.47.mlp.experts.23.gate_proj", "model.layers.47.mlp.experts.24.gate_proj", "model.layers.47.mlp.experts.25.gate_proj", "model.layers.47.mlp.experts.26.gate_proj", "model.layers.47.mlp.experts.27.gate_proj", "model.layers.47.mlp.experts.28.gate_proj", "model.layers.47.mlp.experts.29.gate_proj", "model.layers.47.mlp.experts.30.gate_proj", "model.layers.47.mlp.experts.31.gate_proj", "model.layers.47.mlp.experts.32.gate_proj", "model.layers.47.mlp.experts.33.gate_proj", "model.layers.47.mlp.experts.34.gate_proj", "model.layers.47.mlp.experts.35.gate_proj", "model.layers.47.mlp.experts.36.gate_proj", "model.layers.47.mlp.experts.37.gate_proj", "model.layers.47.mlp.experts.38.gate_proj", "model.layers.47.mlp.experts.39.gate_proj", "model.layers.47.mlp.experts.40.gate_proj", "model.layers.47.mlp.experts.41.gate_proj", "model.layers.47.mlp.experts.42.gate_proj", "model.layers.47.mlp.experts.43.gate_proj", "model.layers.47.mlp.experts.44.gate_proj", "model.layers.47.mlp.experts.45.gate_proj", "model.layers.47.mlp.experts.46.gate_proj", "model.layers.47.mlp.experts.47.gate_proj", "model.layers.47.mlp.experts.48.gate_proj", "model.layers.47.mlp.experts.49.gate_proj", "model.layers.47.mlp.experts.50.gate_proj", "model.layers.47.mlp.experts.51.gate_proj", "model.layers.47.mlp.experts.52.gate_proj", "model.layers.47.mlp.experts.53.gate_proj", "model.layers.47.mlp.experts.54.gate_proj", "model.layers.47.mlp.experts.55.gate_proj", "model.layers.47.mlp.experts.56.gate_proj", "model.layers.47.mlp.experts.57.gate_proj", "model.layers.47.mlp.experts.58.gate_proj", "model.layers.47.mlp.experts.59.gate_proj", "model.layers.47.mlp.experts.60.gate_proj", "model.layers.47.mlp.experts.61.gate_proj", "model.layers.47.mlp.experts.62.gate_proj", "model.layers.47.mlp.experts.63.gate_proj", "model.layers.47.mlp.experts.64.gate_proj", "model.layers.47.mlp.experts.65.gate_proj", "model.layers.47.mlp.experts.66.gate_proj", "model.layers.47.mlp.experts.67.gate_proj", "model.layers.47.mlp.experts.68.gate_proj", "model.layers.47.mlp.experts.69.gate_proj", "model.layers.47.mlp.experts.70.gate_proj", "model.layers.47.mlp.experts.71.gate_proj", "model.layers.47.mlp.experts.72.gate_proj", "model.layers.47.mlp.experts.73.gate_proj", "model.layers.47.mlp.experts.74.gate_proj", "model.layers.47.mlp.experts.75.gate_proj", "model.layers.47.mlp.experts.76.gate_proj", "model.layers.47.mlp.experts.77.gate_proj", "model.layers.47.mlp.experts.78.gate_proj", "model.layers.47.mlp.experts.79.gate_proj", "model.layers.47.mlp.experts.80.gate_proj", "model.layers.47.mlp.experts.81.gate_proj", "model.layers.47.mlp.experts.82.gate_proj", "model.layers.47.mlp.experts.83.gate_proj", "model.layers.47.mlp.experts.84.gate_proj", "model.layers.47.mlp.experts.85.gate_proj", "model.layers.47.mlp.experts.86.gate_proj", "model.layers.47.mlp.experts.87.gate_proj", "model.layers.47.mlp.experts.88.gate_proj", "model.layers.47.mlp.experts.89.gate_proj", "model.layers.47.mlp.experts.90.gate_proj", "model.layers.47.mlp.experts.91.gate_proj", "model.layers.47.mlp.experts.92.gate_proj", "model.layers.47.mlp.experts.93.gate_proj", "model.layers.47.mlp.experts.94.gate_proj", "model.layers.47.mlp.experts.95.gate_proj", "model.layers.47.mlp.experts.96.gate_proj", "model.layers.47.mlp.experts.97.gate_proj", "model.layers.47.mlp.experts.98.gate_proj", "model.layers.47.mlp.experts.99.gate_proj", "model.layers.47.mlp.experts.100.gate_proj", "model.layers.47.mlp.experts.101.gate_proj", "model.layers.47.mlp.experts.102.gate_proj", "model.layers.47.mlp.experts.103.gate_proj", "model.layers.47.mlp.experts.104.gate_proj", "model.layers.47.mlp.experts.105.gate_proj", "model.layers.47.mlp.experts.106.gate_proj", "model.layers.47.mlp.experts.107.gate_proj", "model.layers.47.mlp.experts.108.gate_proj", "model.layers.47.mlp.experts.109.gate_proj", "model.layers.47.mlp.experts.110.gate_proj", "model.layers.47.mlp.experts.111.gate_proj", "model.layers.47.mlp.experts.112.gate_proj", "model.layers.47.mlp.experts.113.gate_proj", "model.layers.47.mlp.experts.114.gate_proj", "model.layers.47.mlp.experts.115.gate_proj", "model.layers.47.mlp.experts.116.gate_proj", "model.layers.47.mlp.experts.117.gate_proj", "model.layers.47.mlp.experts.118.gate_proj", "model.layers.47.mlp.experts.119.gate_proj", "model.layers.47.mlp.experts.120.gate_proj", "model.layers.47.mlp.experts.121.gate_proj", "model.layers.47.mlp.experts.122.gate_proj", "model.layers.47.mlp.experts.123.gate_proj", "model.layers.47.mlp.experts.124.gate_proj", "model.layers.47.mlp.experts.125.gate_proj", "model.layers.47.mlp.experts.126.gate_proj", "model.layers.47.mlp.experts.127.gate_proj", "model.layers.47.mlp.experts.128.gate_proj", "model.layers.47.mlp.experts.129.gate_proj", "model.layers.47.mlp.experts.130.gate_proj", "model.layers.47.mlp.experts.131.gate_proj", "model.layers.47.mlp.experts.132.gate_proj", "model.layers.47.mlp.experts.133.gate_proj", "model.layers.47.mlp.experts.134.gate_proj", "model.layers.47.mlp.experts.135.gate_proj", "model.layers.47.mlp.experts.136.gate_proj", "model.layers.47.mlp.experts.137.gate_proj", "model.layers.47.mlp.experts.138.gate_proj", "model.layers.47.mlp.experts.139.gate_proj", "model.layers.47.mlp.experts.140.gate_proj", "model.layers.47.mlp.experts.141.gate_proj", "model.layers.47.mlp.experts.142.gate_proj", "model.layers.47.mlp.experts.143.gate_proj", "model.layers.47.mlp.experts.144.gate_proj", "model.layers.47.mlp.experts.145.gate_proj", "model.layers.47.mlp.experts.146.gate_proj", "model.layers.47.mlp.experts.147.gate_proj", "model.layers.47.mlp.experts.148.gate_proj", "model.layers.47.mlp.experts.149.gate_proj", "model.layers.47.mlp.experts.150.gate_proj", "model.layers.47.mlp.experts.151.gate_proj", "model.layers.47.mlp.experts.152.gate_proj", "model.layers.47.mlp.experts.153.gate_proj", "model.layers.47.mlp.experts.154.gate_proj", "model.layers.47.mlp.experts.155.gate_proj", "model.layers.47.mlp.experts.156.gate_proj", "model.layers.47.mlp.experts.157.gate_proj", "model.layers.47.mlp.experts.158.gate_proj", "model.layers.47.mlp.experts.159.gate_proj", "model.layers.47.mlp.experts.0.up_proj", "model.layers.47.mlp.experts.1.up_proj", "model.layers.47.mlp.experts.2.up_proj", "model.layers.47.mlp.experts.3.up_proj", "model.layers.47.mlp.experts.4.up_proj", "model.layers.47.mlp.experts.5.up_proj", "model.layers.47.mlp.experts.6.up_proj", "model.layers.47.mlp.experts.7.up_proj", "model.layers.47.mlp.experts.8.up_proj", "model.layers.47.mlp.experts.9.up_proj", "model.layers.47.mlp.experts.10.up_proj", "model.layers.47.mlp.experts.11.up_proj", "model.layers.47.mlp.experts.12.up_proj", "model.layers.47.mlp.experts.13.up_proj", "model.layers.47.mlp.experts.14.up_proj", "model.layers.47.mlp.experts.15.up_proj", "model.layers.47.mlp.experts.16.up_proj", "model.layers.47.mlp.experts.17.up_proj", "model.layers.47.mlp.experts.18.up_proj", "model.layers.47.mlp.experts.19.up_proj", "model.layers.47.mlp.experts.20.up_proj", "model.layers.47.mlp.experts.21.up_proj", "model.layers.47.mlp.experts.22.up_proj", "model.layers.47.mlp.experts.23.up_proj", "model.layers.47.mlp.experts.24.up_proj", "model.layers.47.mlp.experts.25.up_proj", "model.layers.47.mlp.experts.26.up_proj", "model.layers.47.mlp.experts.27.up_proj", "model.layers.47.mlp.experts.28.up_proj", "model.layers.47.mlp.experts.29.up_proj", "model.layers.47.mlp.experts.30.up_proj", "model.layers.47.mlp.experts.31.up_proj", "model.layers.47.mlp.experts.32.up_proj", "model.layers.47.mlp.experts.33.up_proj", "model.layers.47.mlp.experts.34.up_proj", "model.layers.47.mlp.experts.35.up_proj", "model.layers.47.mlp.experts.36.up_proj", "model.layers.47.mlp.experts.37.up_proj", "model.layers.47.mlp.experts.38.up_proj", "model.layers.47.mlp.experts.39.up_proj", "model.layers.47.mlp.experts.40.up_proj", "model.layers.47.mlp.experts.41.up_proj", "model.layers.47.mlp.experts.42.up_proj", "model.layers.47.mlp.experts.43.up_proj", "model.layers.47.mlp.experts.44.up_proj", "model.layers.47.mlp.experts.45.up_proj", "model.layers.47.mlp.experts.46.up_proj", "model.layers.47.mlp.experts.47.up_proj", "model.layers.47.mlp.experts.48.up_proj", "model.layers.47.mlp.experts.49.up_proj", "model.layers.47.mlp.experts.50.up_proj", "model.layers.47.mlp.experts.51.up_proj", "model.layers.47.mlp.experts.52.up_proj", "model.layers.47.mlp.experts.53.up_proj", "model.layers.47.mlp.experts.54.up_proj", "model.layers.47.mlp.experts.55.up_proj", "model.layers.47.mlp.experts.56.up_proj", "model.layers.47.mlp.experts.57.up_proj", "model.layers.47.mlp.experts.58.up_proj", "model.layers.47.mlp.experts.59.up_proj", "model.layers.47.mlp.experts.60.up_proj", "model.layers.47.mlp.experts.61.up_proj", "model.layers.47.mlp.experts.62.up_proj", "model.layers.47.mlp.experts.63.up_proj", "model.layers.47.mlp.experts.64.up_proj", "model.layers.47.mlp.experts.65.up_proj", "model.layers.47.mlp.experts.66.up_proj", "model.layers.47.mlp.experts.67.up_proj", "model.layers.47.mlp.experts.68.up_proj", "model.layers.47.mlp.experts.69.up_proj", "model.layers.47.mlp.experts.70.up_proj", "model.layers.47.mlp.experts.71.up_proj", "model.layers.47.mlp.experts.72.up_proj", "model.layers.47.mlp.experts.73.up_proj", "model.layers.47.mlp.experts.74.up_proj", "model.layers.47.mlp.experts.75.up_proj", "model.layers.47.mlp.experts.76.up_proj", "model.layers.47.mlp.experts.77.up_proj", "model.layers.47.mlp.experts.78.up_proj", "model.layers.47.mlp.experts.79.up_proj", "model.layers.47.mlp.experts.80.up_proj", "model.layers.47.mlp.experts.81.up_proj", "model.layers.47.mlp.experts.82.up_proj", "model.layers.47.mlp.experts.83.up_proj", "model.layers.47.mlp.experts.84.up_proj", "model.layers.47.mlp.experts.85.up_proj", "model.layers.47.mlp.experts.86.up_proj", "model.layers.47.mlp.experts.87.up_proj", "model.layers.47.mlp.experts.88.up_proj", "model.layers.47.mlp.experts.89.up_proj", "model.layers.47.mlp.experts.90.up_proj", "model.layers.47.mlp.experts.91.up_proj", "model.layers.47.mlp.experts.92.up_proj", "model.layers.47.mlp.experts.93.up_proj", "model.layers.47.mlp.experts.94.up_proj", "model.layers.47.mlp.experts.95.up_proj", "model.layers.47.mlp.experts.96.up_proj", "model.layers.47.mlp.experts.97.up_proj", "model.layers.47.mlp.experts.98.up_proj", "model.layers.47.mlp.experts.99.up_proj", "model.layers.47.mlp.experts.100.up_proj", "model.layers.47.mlp.experts.101.up_proj", "model.layers.47.mlp.experts.102.up_proj", "model.layers.47.mlp.experts.103.up_proj", "model.layers.47.mlp.experts.104.up_proj", "model.layers.47.mlp.experts.105.up_proj", "model.layers.47.mlp.experts.106.up_proj", "model.layers.47.mlp.experts.107.up_proj", "model.layers.47.mlp.experts.108.up_proj", "model.layers.47.mlp.experts.109.up_proj", "model.layers.47.mlp.experts.110.up_proj", "model.layers.47.mlp.experts.111.up_proj", "model.layers.47.mlp.experts.112.up_proj", "model.layers.47.mlp.experts.113.up_proj", "model.layers.47.mlp.experts.114.up_proj", "model.layers.47.mlp.experts.115.up_proj", "model.layers.47.mlp.experts.116.up_proj", "model.layers.47.mlp.experts.117.up_proj", "model.layers.47.mlp.experts.118.up_proj", "model.layers.47.mlp.experts.119.up_proj", "model.layers.47.mlp.experts.120.up_proj", "model.layers.47.mlp.experts.121.up_proj", "model.layers.47.mlp.experts.122.up_proj", "model.layers.47.mlp.experts.123.up_proj", "model.layers.47.mlp.experts.124.up_proj", "model.layers.47.mlp.experts.125.up_proj", "model.layers.47.mlp.experts.126.up_proj", "model.layers.47.mlp.experts.127.up_proj", "model.layers.47.mlp.experts.128.up_proj", "model.layers.47.mlp.experts.129.up_proj", "model.layers.47.mlp.experts.130.up_proj", "model.layers.47.mlp.experts.131.up_proj", "model.layers.47.mlp.experts.132.up_proj", "model.layers.47.mlp.experts.133.up_proj", "model.layers.47.mlp.experts.134.up_proj", "model.layers.47.mlp.experts.135.up_proj", "model.layers.47.mlp.experts.136.up_proj", "model.layers.47.mlp.experts.137.up_proj", "model.layers.47.mlp.experts.138.up_proj", "model.layers.47.mlp.experts.139.up_proj", "model.layers.47.mlp.experts.140.up_proj", "model.layers.47.mlp.experts.141.up_proj", "model.layers.47.mlp.experts.142.up_proj", "model.layers.47.mlp.experts.143.up_proj", "model.layers.47.mlp.experts.144.up_proj", "model.layers.47.mlp.experts.145.up_proj", "model.layers.47.mlp.experts.146.up_proj", "model.layers.47.mlp.experts.147.up_proj", "model.layers.47.mlp.experts.148.up_proj", "model.layers.47.mlp.experts.149.up_proj", "model.layers.47.mlp.experts.150.up_proj", "model.layers.47.mlp.experts.151.up_proj", "model.layers.47.mlp.experts.152.up_proj", "model.layers.47.mlp.experts.153.up_proj", "model.layers.47.mlp.experts.154.up_proj", "model.layers.47.mlp.experts.155.up_proj", "model.layers.47.mlp.experts.156.up_proj", "model.layers.47.mlp.experts.157.up_proj", "model.layers.47.mlp.experts.158.up_proj", "model.layers.47.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.131202775985436e-07, "dbits": 5033164800 } ] }, { "idx": 284, "layers": [ "model.layers.47.mlp.experts.0.down_proj", "model.layers.47.mlp.experts.1.down_proj", "model.layers.47.mlp.experts.2.down_proj", "model.layers.47.mlp.experts.3.down_proj", "model.layers.47.mlp.experts.4.down_proj", "model.layers.47.mlp.experts.5.down_proj", "model.layers.47.mlp.experts.6.down_proj", "model.layers.47.mlp.experts.7.down_proj", "model.layers.47.mlp.experts.8.down_proj", "model.layers.47.mlp.experts.9.down_proj", "model.layers.47.mlp.experts.10.down_proj", "model.layers.47.mlp.experts.11.down_proj", "model.layers.47.mlp.experts.12.down_proj", "model.layers.47.mlp.experts.13.down_proj", "model.layers.47.mlp.experts.14.down_proj", "model.layers.47.mlp.experts.15.down_proj", "model.layers.47.mlp.experts.16.down_proj", "model.layers.47.mlp.experts.17.down_proj", "model.layers.47.mlp.experts.18.down_proj", "model.layers.47.mlp.experts.19.down_proj", "model.layers.47.mlp.experts.20.down_proj", "model.layers.47.mlp.experts.21.down_proj", "model.layers.47.mlp.experts.22.down_proj", "model.layers.47.mlp.experts.23.down_proj", "model.layers.47.mlp.experts.24.down_proj", "model.layers.47.mlp.experts.25.down_proj", "model.layers.47.mlp.experts.26.down_proj", "model.layers.47.mlp.experts.27.down_proj", "model.layers.47.mlp.experts.28.down_proj", "model.layers.47.mlp.experts.29.down_proj", "model.layers.47.mlp.experts.30.down_proj", "model.layers.47.mlp.experts.31.down_proj", "model.layers.47.mlp.experts.32.down_proj", "model.layers.47.mlp.experts.33.down_proj", "model.layers.47.mlp.experts.34.down_proj", "model.layers.47.mlp.experts.35.down_proj", "model.layers.47.mlp.experts.36.down_proj", "model.layers.47.mlp.experts.37.down_proj", "model.layers.47.mlp.experts.38.down_proj", "model.layers.47.mlp.experts.39.down_proj", "model.layers.47.mlp.experts.40.down_proj", "model.layers.47.mlp.experts.41.down_proj", "model.layers.47.mlp.experts.42.down_proj", "model.layers.47.mlp.experts.43.down_proj", "model.layers.47.mlp.experts.44.down_proj", "model.layers.47.mlp.experts.45.down_proj", "model.layers.47.mlp.experts.46.down_proj", "model.layers.47.mlp.experts.47.down_proj", "model.layers.47.mlp.experts.48.down_proj", "model.layers.47.mlp.experts.49.down_proj", "model.layers.47.mlp.experts.50.down_proj", "model.layers.47.mlp.experts.51.down_proj", "model.layers.47.mlp.experts.52.down_proj", "model.layers.47.mlp.experts.53.down_proj", "model.layers.47.mlp.experts.54.down_proj", "model.layers.47.mlp.experts.55.down_proj", "model.layers.47.mlp.experts.56.down_proj", "model.layers.47.mlp.experts.57.down_proj", "model.layers.47.mlp.experts.58.down_proj", "model.layers.47.mlp.experts.59.down_proj", "model.layers.47.mlp.experts.60.down_proj", "model.layers.47.mlp.experts.61.down_proj", "model.layers.47.mlp.experts.62.down_proj", "model.layers.47.mlp.experts.63.down_proj", "model.layers.47.mlp.experts.64.down_proj", "model.layers.47.mlp.experts.65.down_proj", "model.layers.47.mlp.experts.66.down_proj", "model.layers.47.mlp.experts.67.down_proj", "model.layers.47.mlp.experts.68.down_proj", "model.layers.47.mlp.experts.69.down_proj", "model.layers.47.mlp.experts.70.down_proj", "model.layers.47.mlp.experts.71.down_proj", "model.layers.47.mlp.experts.72.down_proj", "model.layers.47.mlp.experts.73.down_proj", "model.layers.47.mlp.experts.74.down_proj", "model.layers.47.mlp.experts.75.down_proj", "model.layers.47.mlp.experts.76.down_proj", "model.layers.47.mlp.experts.77.down_proj", "model.layers.47.mlp.experts.78.down_proj", "model.layers.47.mlp.experts.79.down_proj", "model.layers.47.mlp.experts.80.down_proj", "model.layers.47.mlp.experts.81.down_proj", "model.layers.47.mlp.experts.82.down_proj", "model.layers.47.mlp.experts.83.down_proj", "model.layers.47.mlp.experts.84.down_proj", "model.layers.47.mlp.experts.85.down_proj", "model.layers.47.mlp.experts.86.down_proj", "model.layers.47.mlp.experts.87.down_proj", "model.layers.47.mlp.experts.88.down_proj", "model.layers.47.mlp.experts.89.down_proj", "model.layers.47.mlp.experts.90.down_proj", "model.layers.47.mlp.experts.91.down_proj", "model.layers.47.mlp.experts.92.down_proj", "model.layers.47.mlp.experts.93.down_proj", "model.layers.47.mlp.experts.94.down_proj", "model.layers.47.mlp.experts.95.down_proj", "model.layers.47.mlp.experts.96.down_proj", "model.layers.47.mlp.experts.97.down_proj", "model.layers.47.mlp.experts.98.down_proj", "model.layers.47.mlp.experts.99.down_proj", "model.layers.47.mlp.experts.100.down_proj", "model.layers.47.mlp.experts.101.down_proj", "model.layers.47.mlp.experts.102.down_proj", "model.layers.47.mlp.experts.103.down_proj", "model.layers.47.mlp.experts.104.down_proj", "model.layers.47.mlp.experts.105.down_proj", "model.layers.47.mlp.experts.106.down_proj", "model.layers.47.mlp.experts.107.down_proj", "model.layers.47.mlp.experts.108.down_proj", "model.layers.47.mlp.experts.109.down_proj", "model.layers.47.mlp.experts.110.down_proj", "model.layers.47.mlp.experts.111.down_proj", "model.layers.47.mlp.experts.112.down_proj", "model.layers.47.mlp.experts.113.down_proj", "model.layers.47.mlp.experts.114.down_proj", "model.layers.47.mlp.experts.115.down_proj", "model.layers.47.mlp.experts.116.down_proj", "model.layers.47.mlp.experts.117.down_proj", "model.layers.47.mlp.experts.118.down_proj", "model.layers.47.mlp.experts.119.down_proj", "model.layers.47.mlp.experts.120.down_proj", "model.layers.47.mlp.experts.121.down_proj", "model.layers.47.mlp.experts.122.down_proj", "model.layers.47.mlp.experts.123.down_proj", "model.layers.47.mlp.experts.124.down_proj", "model.layers.47.mlp.experts.125.down_proj", "model.layers.47.mlp.experts.126.down_proj", "model.layers.47.mlp.experts.127.down_proj", "model.layers.47.mlp.experts.128.down_proj", "model.layers.47.mlp.experts.129.down_proj", "model.layers.47.mlp.experts.130.down_proj", "model.layers.47.mlp.experts.131.down_proj", "model.layers.47.mlp.experts.132.down_proj", "model.layers.47.mlp.experts.133.down_proj", "model.layers.47.mlp.experts.134.down_proj", "model.layers.47.mlp.experts.135.down_proj", "model.layers.47.mlp.experts.136.down_proj", "model.layers.47.mlp.experts.137.down_proj", "model.layers.47.mlp.experts.138.down_proj", "model.layers.47.mlp.experts.139.down_proj", "model.layers.47.mlp.experts.140.down_proj", "model.layers.47.mlp.experts.141.down_proj", "model.layers.47.mlp.experts.142.down_proj", "model.layers.47.mlp.experts.143.down_proj", "model.layers.47.mlp.experts.144.down_proj", "model.layers.47.mlp.experts.145.down_proj", "model.layers.47.mlp.experts.146.down_proj", "model.layers.47.mlp.experts.147.down_proj", "model.layers.47.mlp.experts.148.down_proj", "model.layers.47.mlp.experts.149.down_proj", "model.layers.47.mlp.experts.150.down_proj", "model.layers.47.mlp.experts.151.down_proj", "model.layers.47.mlp.experts.152.down_proj", "model.layers.47.mlp.experts.153.down_proj", "model.layers.47.mlp.experts.154.down_proj", "model.layers.47.mlp.experts.155.down_proj", "model.layers.47.mlp.experts.156.down_proj", "model.layers.47.mlp.experts.157.down_proj", "model.layers.47.mlp.experts.158.down_proj", "model.layers.47.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -6.102491170170091e-08, "dbits": 2516582400 } ] }, { "idx": 285, "layers": [ "model.layers.48.self_attn.q_proj" ], "candidates": [ { "dkld": 2.5431509129703045e-06, "dbits": 125829120 } ] }, { "idx": 286, "layers": [ "model.layers.48.self_attn.k_proj", "model.layers.48.self_attn.v_proj" ], "candidates": [ { "dkld": 4.583131521940318e-06, "dbits": 20971520 } ] }, { "idx": 287, "layers": [ "model.layers.48.self_attn.o_proj" ], "candidates": [ { "dkld": 4.607730079442219e-06, "dbits": 125829120 } ] }, { "idx": 288, "layers": [ "model.layers.48.mlp.shared_experts.gate_proj", "model.layers.48.mlp.shared_experts.up_proj", "model.layers.48.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.127690822817342e-06, "dbits": 47185920 } ] }, { "idx": 289, "layers": [ "model.layers.48.mlp.experts.0.gate_proj", "model.layers.48.mlp.experts.1.gate_proj", "model.layers.48.mlp.experts.2.gate_proj", "model.layers.48.mlp.experts.3.gate_proj", "model.layers.48.mlp.experts.4.gate_proj", "model.layers.48.mlp.experts.5.gate_proj", "model.layers.48.mlp.experts.6.gate_proj", "model.layers.48.mlp.experts.7.gate_proj", "model.layers.48.mlp.experts.8.gate_proj", "model.layers.48.mlp.experts.9.gate_proj", "model.layers.48.mlp.experts.10.gate_proj", "model.layers.48.mlp.experts.11.gate_proj", "model.layers.48.mlp.experts.12.gate_proj", "model.layers.48.mlp.experts.13.gate_proj", "model.layers.48.mlp.experts.14.gate_proj", "model.layers.48.mlp.experts.15.gate_proj", "model.layers.48.mlp.experts.16.gate_proj", "model.layers.48.mlp.experts.17.gate_proj", "model.layers.48.mlp.experts.18.gate_proj", "model.layers.48.mlp.experts.19.gate_proj", "model.layers.48.mlp.experts.20.gate_proj", "model.layers.48.mlp.experts.21.gate_proj", "model.layers.48.mlp.experts.22.gate_proj", "model.layers.48.mlp.experts.23.gate_proj", "model.layers.48.mlp.experts.24.gate_proj", "model.layers.48.mlp.experts.25.gate_proj", "model.layers.48.mlp.experts.26.gate_proj", "model.layers.48.mlp.experts.27.gate_proj", "model.layers.48.mlp.experts.28.gate_proj", "model.layers.48.mlp.experts.29.gate_proj", "model.layers.48.mlp.experts.30.gate_proj", "model.layers.48.mlp.experts.31.gate_proj", "model.layers.48.mlp.experts.32.gate_proj", "model.layers.48.mlp.experts.33.gate_proj", "model.layers.48.mlp.experts.34.gate_proj", "model.layers.48.mlp.experts.35.gate_proj", "model.layers.48.mlp.experts.36.gate_proj", "model.layers.48.mlp.experts.37.gate_proj", "model.layers.48.mlp.experts.38.gate_proj", "model.layers.48.mlp.experts.39.gate_proj", "model.layers.48.mlp.experts.40.gate_proj", "model.layers.48.mlp.experts.41.gate_proj", "model.layers.48.mlp.experts.42.gate_proj", "model.layers.48.mlp.experts.43.gate_proj", "model.layers.48.mlp.experts.44.gate_proj", "model.layers.48.mlp.experts.45.gate_proj", "model.layers.48.mlp.experts.46.gate_proj", "model.layers.48.mlp.experts.47.gate_proj", "model.layers.48.mlp.experts.48.gate_proj", "model.layers.48.mlp.experts.49.gate_proj", "model.layers.48.mlp.experts.50.gate_proj", "model.layers.48.mlp.experts.51.gate_proj", "model.layers.48.mlp.experts.52.gate_proj", "model.layers.48.mlp.experts.53.gate_proj", "model.layers.48.mlp.experts.54.gate_proj", "model.layers.48.mlp.experts.55.gate_proj", "model.layers.48.mlp.experts.56.gate_proj", "model.layers.48.mlp.experts.57.gate_proj", "model.layers.48.mlp.experts.58.gate_proj", "model.layers.48.mlp.experts.59.gate_proj", "model.layers.48.mlp.experts.60.gate_proj", "model.layers.48.mlp.experts.61.gate_proj", "model.layers.48.mlp.experts.62.gate_proj", "model.layers.48.mlp.experts.63.gate_proj", "model.layers.48.mlp.experts.64.gate_proj", "model.layers.48.mlp.experts.65.gate_proj", "model.layers.48.mlp.experts.66.gate_proj", "model.layers.48.mlp.experts.67.gate_proj", "model.layers.48.mlp.experts.68.gate_proj", "model.layers.48.mlp.experts.69.gate_proj", "model.layers.48.mlp.experts.70.gate_proj", "model.layers.48.mlp.experts.71.gate_proj", "model.layers.48.mlp.experts.72.gate_proj", "model.layers.48.mlp.experts.73.gate_proj", "model.layers.48.mlp.experts.74.gate_proj", "model.layers.48.mlp.experts.75.gate_proj", "model.layers.48.mlp.experts.76.gate_proj", "model.layers.48.mlp.experts.77.gate_proj", "model.layers.48.mlp.experts.78.gate_proj", "model.layers.48.mlp.experts.79.gate_proj", "model.layers.48.mlp.experts.80.gate_proj", "model.layers.48.mlp.experts.81.gate_proj", "model.layers.48.mlp.experts.82.gate_proj", "model.layers.48.mlp.experts.83.gate_proj", "model.layers.48.mlp.experts.84.gate_proj", "model.layers.48.mlp.experts.85.gate_proj", "model.layers.48.mlp.experts.86.gate_proj", "model.layers.48.mlp.experts.87.gate_proj", "model.layers.48.mlp.experts.88.gate_proj", "model.layers.48.mlp.experts.89.gate_proj", "model.layers.48.mlp.experts.90.gate_proj", "model.layers.48.mlp.experts.91.gate_proj", "model.layers.48.mlp.experts.92.gate_proj", "model.layers.48.mlp.experts.93.gate_proj", "model.layers.48.mlp.experts.94.gate_proj", "model.layers.48.mlp.experts.95.gate_proj", "model.layers.48.mlp.experts.96.gate_proj", "model.layers.48.mlp.experts.97.gate_proj", "model.layers.48.mlp.experts.98.gate_proj", "model.layers.48.mlp.experts.99.gate_proj", "model.layers.48.mlp.experts.100.gate_proj", "model.layers.48.mlp.experts.101.gate_proj", "model.layers.48.mlp.experts.102.gate_proj", "model.layers.48.mlp.experts.103.gate_proj", "model.layers.48.mlp.experts.104.gate_proj", "model.layers.48.mlp.experts.105.gate_proj", "model.layers.48.mlp.experts.106.gate_proj", "model.layers.48.mlp.experts.107.gate_proj", "model.layers.48.mlp.experts.108.gate_proj", "model.layers.48.mlp.experts.109.gate_proj", "model.layers.48.mlp.experts.110.gate_proj", "model.layers.48.mlp.experts.111.gate_proj", "model.layers.48.mlp.experts.112.gate_proj", "model.layers.48.mlp.experts.113.gate_proj", "model.layers.48.mlp.experts.114.gate_proj", "model.layers.48.mlp.experts.115.gate_proj", "model.layers.48.mlp.experts.116.gate_proj", "model.layers.48.mlp.experts.117.gate_proj", "model.layers.48.mlp.experts.118.gate_proj", "model.layers.48.mlp.experts.119.gate_proj", "model.layers.48.mlp.experts.120.gate_proj", "model.layers.48.mlp.experts.121.gate_proj", "model.layers.48.mlp.experts.122.gate_proj", "model.layers.48.mlp.experts.123.gate_proj", "model.layers.48.mlp.experts.124.gate_proj", "model.layers.48.mlp.experts.125.gate_proj", "model.layers.48.mlp.experts.126.gate_proj", "model.layers.48.mlp.experts.127.gate_proj", "model.layers.48.mlp.experts.128.gate_proj", "model.layers.48.mlp.experts.129.gate_proj", "model.layers.48.mlp.experts.130.gate_proj", "model.layers.48.mlp.experts.131.gate_proj", "model.layers.48.mlp.experts.132.gate_proj", "model.layers.48.mlp.experts.133.gate_proj", "model.layers.48.mlp.experts.134.gate_proj", "model.layers.48.mlp.experts.135.gate_proj", "model.layers.48.mlp.experts.136.gate_proj", "model.layers.48.mlp.experts.137.gate_proj", "model.layers.48.mlp.experts.138.gate_proj", "model.layers.48.mlp.experts.139.gate_proj", "model.layers.48.mlp.experts.140.gate_proj", "model.layers.48.mlp.experts.141.gate_proj", "model.layers.48.mlp.experts.142.gate_proj", "model.layers.48.mlp.experts.143.gate_proj", "model.layers.48.mlp.experts.144.gate_proj", "model.layers.48.mlp.experts.145.gate_proj", "model.layers.48.mlp.experts.146.gate_proj", "model.layers.48.mlp.experts.147.gate_proj", "model.layers.48.mlp.experts.148.gate_proj", "model.layers.48.mlp.experts.149.gate_proj", "model.layers.48.mlp.experts.150.gate_proj", "model.layers.48.mlp.experts.151.gate_proj", "model.layers.48.mlp.experts.152.gate_proj", "model.layers.48.mlp.experts.153.gate_proj", "model.layers.48.mlp.experts.154.gate_proj", "model.layers.48.mlp.experts.155.gate_proj", "model.layers.48.mlp.experts.156.gate_proj", "model.layers.48.mlp.experts.157.gate_proj", "model.layers.48.mlp.experts.158.gate_proj", "model.layers.48.mlp.experts.159.gate_proj", "model.layers.48.mlp.experts.0.up_proj", "model.layers.48.mlp.experts.1.up_proj", "model.layers.48.mlp.experts.2.up_proj", "model.layers.48.mlp.experts.3.up_proj", "model.layers.48.mlp.experts.4.up_proj", "model.layers.48.mlp.experts.5.up_proj", "model.layers.48.mlp.experts.6.up_proj", "model.layers.48.mlp.experts.7.up_proj", "model.layers.48.mlp.experts.8.up_proj", "model.layers.48.mlp.experts.9.up_proj", "model.layers.48.mlp.experts.10.up_proj", "model.layers.48.mlp.experts.11.up_proj", "model.layers.48.mlp.experts.12.up_proj", "model.layers.48.mlp.experts.13.up_proj", "model.layers.48.mlp.experts.14.up_proj", "model.layers.48.mlp.experts.15.up_proj", "model.layers.48.mlp.experts.16.up_proj", "model.layers.48.mlp.experts.17.up_proj", "model.layers.48.mlp.experts.18.up_proj", "model.layers.48.mlp.experts.19.up_proj", "model.layers.48.mlp.experts.20.up_proj", "model.layers.48.mlp.experts.21.up_proj", "model.layers.48.mlp.experts.22.up_proj", "model.layers.48.mlp.experts.23.up_proj", "model.layers.48.mlp.experts.24.up_proj", "model.layers.48.mlp.experts.25.up_proj", "model.layers.48.mlp.experts.26.up_proj", "model.layers.48.mlp.experts.27.up_proj", "model.layers.48.mlp.experts.28.up_proj", "model.layers.48.mlp.experts.29.up_proj", "model.layers.48.mlp.experts.30.up_proj", "model.layers.48.mlp.experts.31.up_proj", "model.layers.48.mlp.experts.32.up_proj", "model.layers.48.mlp.experts.33.up_proj", "model.layers.48.mlp.experts.34.up_proj", "model.layers.48.mlp.experts.35.up_proj", "model.layers.48.mlp.experts.36.up_proj", "model.layers.48.mlp.experts.37.up_proj", "model.layers.48.mlp.experts.38.up_proj", "model.layers.48.mlp.experts.39.up_proj", "model.layers.48.mlp.experts.40.up_proj", "model.layers.48.mlp.experts.41.up_proj", "model.layers.48.mlp.experts.42.up_proj", "model.layers.48.mlp.experts.43.up_proj", "model.layers.48.mlp.experts.44.up_proj", "model.layers.48.mlp.experts.45.up_proj", "model.layers.48.mlp.experts.46.up_proj", "model.layers.48.mlp.experts.47.up_proj", "model.layers.48.mlp.experts.48.up_proj", "model.layers.48.mlp.experts.49.up_proj", "model.layers.48.mlp.experts.50.up_proj", "model.layers.48.mlp.experts.51.up_proj", "model.layers.48.mlp.experts.52.up_proj", "model.layers.48.mlp.experts.53.up_proj", "model.layers.48.mlp.experts.54.up_proj", "model.layers.48.mlp.experts.55.up_proj", "model.layers.48.mlp.experts.56.up_proj", "model.layers.48.mlp.experts.57.up_proj", "model.layers.48.mlp.experts.58.up_proj", "model.layers.48.mlp.experts.59.up_proj", "model.layers.48.mlp.experts.60.up_proj", "model.layers.48.mlp.experts.61.up_proj", "model.layers.48.mlp.experts.62.up_proj", "model.layers.48.mlp.experts.63.up_proj", "model.layers.48.mlp.experts.64.up_proj", "model.layers.48.mlp.experts.65.up_proj", "model.layers.48.mlp.experts.66.up_proj", "model.layers.48.mlp.experts.67.up_proj", "model.layers.48.mlp.experts.68.up_proj", "model.layers.48.mlp.experts.69.up_proj", "model.layers.48.mlp.experts.70.up_proj", "model.layers.48.mlp.experts.71.up_proj", "model.layers.48.mlp.experts.72.up_proj", "model.layers.48.mlp.experts.73.up_proj", "model.layers.48.mlp.experts.74.up_proj", "model.layers.48.mlp.experts.75.up_proj", "model.layers.48.mlp.experts.76.up_proj", "model.layers.48.mlp.experts.77.up_proj", "model.layers.48.mlp.experts.78.up_proj", "model.layers.48.mlp.experts.79.up_proj", "model.layers.48.mlp.experts.80.up_proj", "model.layers.48.mlp.experts.81.up_proj", "model.layers.48.mlp.experts.82.up_proj", "model.layers.48.mlp.experts.83.up_proj", "model.layers.48.mlp.experts.84.up_proj", "model.layers.48.mlp.experts.85.up_proj", "model.layers.48.mlp.experts.86.up_proj", "model.layers.48.mlp.experts.87.up_proj", "model.layers.48.mlp.experts.88.up_proj", "model.layers.48.mlp.experts.89.up_proj", "model.layers.48.mlp.experts.90.up_proj", "model.layers.48.mlp.experts.91.up_proj", "model.layers.48.mlp.experts.92.up_proj", "model.layers.48.mlp.experts.93.up_proj", "model.layers.48.mlp.experts.94.up_proj", "model.layers.48.mlp.experts.95.up_proj", "model.layers.48.mlp.experts.96.up_proj", "model.layers.48.mlp.experts.97.up_proj", "model.layers.48.mlp.experts.98.up_proj", "model.layers.48.mlp.experts.99.up_proj", "model.layers.48.mlp.experts.100.up_proj", "model.layers.48.mlp.experts.101.up_proj", "model.layers.48.mlp.experts.102.up_proj", "model.layers.48.mlp.experts.103.up_proj", "model.layers.48.mlp.experts.104.up_proj", "model.layers.48.mlp.experts.105.up_proj", "model.layers.48.mlp.experts.106.up_proj", "model.layers.48.mlp.experts.107.up_proj", "model.layers.48.mlp.experts.108.up_proj", "model.layers.48.mlp.experts.109.up_proj", "model.layers.48.mlp.experts.110.up_proj", "model.layers.48.mlp.experts.111.up_proj", "model.layers.48.mlp.experts.112.up_proj", "model.layers.48.mlp.experts.113.up_proj", "model.layers.48.mlp.experts.114.up_proj", "model.layers.48.mlp.experts.115.up_proj", "model.layers.48.mlp.experts.116.up_proj", "model.layers.48.mlp.experts.117.up_proj", "model.layers.48.mlp.experts.118.up_proj", "model.layers.48.mlp.experts.119.up_proj", "model.layers.48.mlp.experts.120.up_proj", "model.layers.48.mlp.experts.121.up_proj", "model.layers.48.mlp.experts.122.up_proj", "model.layers.48.mlp.experts.123.up_proj", "model.layers.48.mlp.experts.124.up_proj", "model.layers.48.mlp.experts.125.up_proj", "model.layers.48.mlp.experts.126.up_proj", "model.layers.48.mlp.experts.127.up_proj", "model.layers.48.mlp.experts.128.up_proj", "model.layers.48.mlp.experts.129.up_proj", "model.layers.48.mlp.experts.130.up_proj", "model.layers.48.mlp.experts.131.up_proj", "model.layers.48.mlp.experts.132.up_proj", "model.layers.48.mlp.experts.133.up_proj", "model.layers.48.mlp.experts.134.up_proj", "model.layers.48.mlp.experts.135.up_proj", "model.layers.48.mlp.experts.136.up_proj", "model.layers.48.mlp.experts.137.up_proj", "model.layers.48.mlp.experts.138.up_proj", "model.layers.48.mlp.experts.139.up_proj", "model.layers.48.mlp.experts.140.up_proj", "model.layers.48.mlp.experts.141.up_proj", "model.layers.48.mlp.experts.142.up_proj", "model.layers.48.mlp.experts.143.up_proj", "model.layers.48.mlp.experts.144.up_proj", "model.layers.48.mlp.experts.145.up_proj", "model.layers.48.mlp.experts.146.up_proj", "model.layers.48.mlp.experts.147.up_proj", "model.layers.48.mlp.experts.148.up_proj", "model.layers.48.mlp.experts.149.up_proj", "model.layers.48.mlp.experts.150.up_proj", "model.layers.48.mlp.experts.151.up_proj", "model.layers.48.mlp.experts.152.up_proj", "model.layers.48.mlp.experts.153.up_proj", "model.layers.48.mlp.experts.154.up_proj", "model.layers.48.mlp.experts.155.up_proj", "model.layers.48.mlp.experts.156.up_proj", "model.layers.48.mlp.experts.157.up_proj", "model.layers.48.mlp.experts.158.up_proj", "model.layers.48.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.0605086572468498e-06, "dbits": 5033164800 } ] }, { "idx": 290, "layers": [ "model.layers.48.mlp.experts.0.down_proj", "model.layers.48.mlp.experts.1.down_proj", "model.layers.48.mlp.experts.2.down_proj", "model.layers.48.mlp.experts.3.down_proj", "model.layers.48.mlp.experts.4.down_proj", "model.layers.48.mlp.experts.5.down_proj", "model.layers.48.mlp.experts.6.down_proj", "model.layers.48.mlp.experts.7.down_proj", "model.layers.48.mlp.experts.8.down_proj", "model.layers.48.mlp.experts.9.down_proj", "model.layers.48.mlp.experts.10.down_proj", "model.layers.48.mlp.experts.11.down_proj", "model.layers.48.mlp.experts.12.down_proj", "model.layers.48.mlp.experts.13.down_proj", "model.layers.48.mlp.experts.14.down_proj", "model.layers.48.mlp.experts.15.down_proj", "model.layers.48.mlp.experts.16.down_proj", "model.layers.48.mlp.experts.17.down_proj", "model.layers.48.mlp.experts.18.down_proj", "model.layers.48.mlp.experts.19.down_proj", "model.layers.48.mlp.experts.20.down_proj", "model.layers.48.mlp.experts.21.down_proj", "model.layers.48.mlp.experts.22.down_proj", "model.layers.48.mlp.experts.23.down_proj", "model.layers.48.mlp.experts.24.down_proj", "model.layers.48.mlp.experts.25.down_proj", "model.layers.48.mlp.experts.26.down_proj", "model.layers.48.mlp.experts.27.down_proj", "model.layers.48.mlp.experts.28.down_proj", "model.layers.48.mlp.experts.29.down_proj", "model.layers.48.mlp.experts.30.down_proj", "model.layers.48.mlp.experts.31.down_proj", "model.layers.48.mlp.experts.32.down_proj", "model.layers.48.mlp.experts.33.down_proj", "model.layers.48.mlp.experts.34.down_proj", "model.layers.48.mlp.experts.35.down_proj", "model.layers.48.mlp.experts.36.down_proj", "model.layers.48.mlp.experts.37.down_proj", "model.layers.48.mlp.experts.38.down_proj", "model.layers.48.mlp.experts.39.down_proj", "model.layers.48.mlp.experts.40.down_proj", "model.layers.48.mlp.experts.41.down_proj", "model.layers.48.mlp.experts.42.down_proj", "model.layers.48.mlp.experts.43.down_proj", "model.layers.48.mlp.experts.44.down_proj", "model.layers.48.mlp.experts.45.down_proj", "model.layers.48.mlp.experts.46.down_proj", "model.layers.48.mlp.experts.47.down_proj", "model.layers.48.mlp.experts.48.down_proj", "model.layers.48.mlp.experts.49.down_proj", "model.layers.48.mlp.experts.50.down_proj", "model.layers.48.mlp.experts.51.down_proj", "model.layers.48.mlp.experts.52.down_proj", "model.layers.48.mlp.experts.53.down_proj", "model.layers.48.mlp.experts.54.down_proj", "model.layers.48.mlp.experts.55.down_proj", "model.layers.48.mlp.experts.56.down_proj", "model.layers.48.mlp.experts.57.down_proj", "model.layers.48.mlp.experts.58.down_proj", "model.layers.48.mlp.experts.59.down_proj", "model.layers.48.mlp.experts.60.down_proj", "model.layers.48.mlp.experts.61.down_proj", "model.layers.48.mlp.experts.62.down_proj", "model.layers.48.mlp.experts.63.down_proj", "model.layers.48.mlp.experts.64.down_proj", "model.layers.48.mlp.experts.65.down_proj", "model.layers.48.mlp.experts.66.down_proj", "model.layers.48.mlp.experts.67.down_proj", "model.layers.48.mlp.experts.68.down_proj", "model.layers.48.mlp.experts.69.down_proj", "model.layers.48.mlp.experts.70.down_proj", "model.layers.48.mlp.experts.71.down_proj", "model.layers.48.mlp.experts.72.down_proj", "model.layers.48.mlp.experts.73.down_proj", "model.layers.48.mlp.experts.74.down_proj", "model.layers.48.mlp.experts.75.down_proj", "model.layers.48.mlp.experts.76.down_proj", "model.layers.48.mlp.experts.77.down_proj", "model.layers.48.mlp.experts.78.down_proj", "model.layers.48.mlp.experts.79.down_proj", "model.layers.48.mlp.experts.80.down_proj", "model.layers.48.mlp.experts.81.down_proj", "model.layers.48.mlp.experts.82.down_proj", "model.layers.48.mlp.experts.83.down_proj", "model.layers.48.mlp.experts.84.down_proj", "model.layers.48.mlp.experts.85.down_proj", "model.layers.48.mlp.experts.86.down_proj", "model.layers.48.mlp.experts.87.down_proj", "model.layers.48.mlp.experts.88.down_proj", "model.layers.48.mlp.experts.89.down_proj", "model.layers.48.mlp.experts.90.down_proj", "model.layers.48.mlp.experts.91.down_proj", "model.layers.48.mlp.experts.92.down_proj", "model.layers.48.mlp.experts.93.down_proj", "model.layers.48.mlp.experts.94.down_proj", "model.layers.48.mlp.experts.95.down_proj", "model.layers.48.mlp.experts.96.down_proj", "model.layers.48.mlp.experts.97.down_proj", "model.layers.48.mlp.experts.98.down_proj", "model.layers.48.mlp.experts.99.down_proj", "model.layers.48.mlp.experts.100.down_proj", "model.layers.48.mlp.experts.101.down_proj", "model.layers.48.mlp.experts.102.down_proj", "model.layers.48.mlp.experts.103.down_proj", "model.layers.48.mlp.experts.104.down_proj", "model.layers.48.mlp.experts.105.down_proj", "model.layers.48.mlp.experts.106.down_proj", "model.layers.48.mlp.experts.107.down_proj", "model.layers.48.mlp.experts.108.down_proj", "model.layers.48.mlp.experts.109.down_proj", "model.layers.48.mlp.experts.110.down_proj", "model.layers.48.mlp.experts.111.down_proj", "model.layers.48.mlp.experts.112.down_proj", "model.layers.48.mlp.experts.113.down_proj", "model.layers.48.mlp.experts.114.down_proj", "model.layers.48.mlp.experts.115.down_proj", "model.layers.48.mlp.experts.116.down_proj", "model.layers.48.mlp.experts.117.down_proj", "model.layers.48.mlp.experts.118.down_proj", "model.layers.48.mlp.experts.119.down_proj", "model.layers.48.mlp.experts.120.down_proj", "model.layers.48.mlp.experts.121.down_proj", "model.layers.48.mlp.experts.122.down_proj", "model.layers.48.mlp.experts.123.down_proj", "model.layers.48.mlp.experts.124.down_proj", "model.layers.48.mlp.experts.125.down_proj", "model.layers.48.mlp.experts.126.down_proj", "model.layers.48.mlp.experts.127.down_proj", "model.layers.48.mlp.experts.128.down_proj", "model.layers.48.mlp.experts.129.down_proj", "model.layers.48.mlp.experts.130.down_proj", "model.layers.48.mlp.experts.131.down_proj", "model.layers.48.mlp.experts.132.down_proj", "model.layers.48.mlp.experts.133.down_proj", "model.layers.48.mlp.experts.134.down_proj", "model.layers.48.mlp.experts.135.down_proj", "model.layers.48.mlp.experts.136.down_proj", "model.layers.48.mlp.experts.137.down_proj", "model.layers.48.mlp.experts.138.down_proj", "model.layers.48.mlp.experts.139.down_proj", "model.layers.48.mlp.experts.140.down_proj", "model.layers.48.mlp.experts.141.down_proj", "model.layers.48.mlp.experts.142.down_proj", "model.layers.48.mlp.experts.143.down_proj", "model.layers.48.mlp.experts.144.down_proj", "model.layers.48.mlp.experts.145.down_proj", "model.layers.48.mlp.experts.146.down_proj", "model.layers.48.mlp.experts.147.down_proj", "model.layers.48.mlp.experts.148.down_proj", "model.layers.48.mlp.experts.149.down_proj", "model.layers.48.mlp.experts.150.down_proj", "model.layers.48.mlp.experts.151.down_proj", "model.layers.48.mlp.experts.152.down_proj", "model.layers.48.mlp.experts.153.down_proj", "model.layers.48.mlp.experts.154.down_proj", "model.layers.48.mlp.experts.155.down_proj", "model.layers.48.mlp.experts.156.down_proj", "model.layers.48.mlp.experts.157.down_proj", "model.layers.48.mlp.experts.158.down_proj", "model.layers.48.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 5.490757757798271e-07, "dbits": 2516582400 } ] }, { "idx": 291, "layers": [ "model.layers.49.self_attn.q_proj" ], "candidates": [ { "dkld": 3.092893166467981e-07, "dbits": 125829120 } ] }, { "idx": 292, "layers": [ "model.layers.49.self_attn.k_proj", "model.layers.49.self_attn.v_proj" ], "candidates": [ { "dkld": 2.629571827128486e-06, "dbits": 20971520 } ] }, { "idx": 293, "layers": [ "model.layers.49.self_attn.o_proj" ], "candidates": [ { "dkld": 7.375114364549301e-07, "dbits": 125829120 } ] }, { "idx": 294, "layers": [ "model.layers.49.mlp.shared_experts.gate_proj", "model.layers.49.mlp.shared_experts.up_proj", "model.layers.49.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.1143176127224727e-06, "dbits": 47185920 } ] }, { "idx": 295, "layers": [ "model.layers.49.mlp.experts.0.gate_proj", "model.layers.49.mlp.experts.1.gate_proj", "model.layers.49.mlp.experts.2.gate_proj", "model.layers.49.mlp.experts.3.gate_proj", "model.layers.49.mlp.experts.4.gate_proj", "model.layers.49.mlp.experts.5.gate_proj", "model.layers.49.mlp.experts.6.gate_proj", "model.layers.49.mlp.experts.7.gate_proj", "model.layers.49.mlp.experts.8.gate_proj", "model.layers.49.mlp.experts.9.gate_proj", "model.layers.49.mlp.experts.10.gate_proj", "model.layers.49.mlp.experts.11.gate_proj", "model.layers.49.mlp.experts.12.gate_proj", "model.layers.49.mlp.experts.13.gate_proj", "model.layers.49.mlp.experts.14.gate_proj", "model.layers.49.mlp.experts.15.gate_proj", "model.layers.49.mlp.experts.16.gate_proj", "model.layers.49.mlp.experts.17.gate_proj", "model.layers.49.mlp.experts.18.gate_proj", "model.layers.49.mlp.experts.19.gate_proj", "model.layers.49.mlp.experts.20.gate_proj", "model.layers.49.mlp.experts.21.gate_proj", "model.layers.49.mlp.experts.22.gate_proj", "model.layers.49.mlp.experts.23.gate_proj", "model.layers.49.mlp.experts.24.gate_proj", "model.layers.49.mlp.experts.25.gate_proj", "model.layers.49.mlp.experts.26.gate_proj", "model.layers.49.mlp.experts.27.gate_proj", "model.layers.49.mlp.experts.28.gate_proj", "model.layers.49.mlp.experts.29.gate_proj", "model.layers.49.mlp.experts.30.gate_proj", "model.layers.49.mlp.experts.31.gate_proj", "model.layers.49.mlp.experts.32.gate_proj", "model.layers.49.mlp.experts.33.gate_proj", "model.layers.49.mlp.experts.34.gate_proj", "model.layers.49.mlp.experts.35.gate_proj", "model.layers.49.mlp.experts.36.gate_proj", "model.layers.49.mlp.experts.37.gate_proj", "model.layers.49.mlp.experts.38.gate_proj", "model.layers.49.mlp.experts.39.gate_proj", "model.layers.49.mlp.experts.40.gate_proj", "model.layers.49.mlp.experts.41.gate_proj", "model.layers.49.mlp.experts.42.gate_proj", "model.layers.49.mlp.experts.43.gate_proj", "model.layers.49.mlp.experts.44.gate_proj", "model.layers.49.mlp.experts.45.gate_proj", "model.layers.49.mlp.experts.46.gate_proj", "model.layers.49.mlp.experts.47.gate_proj", "model.layers.49.mlp.experts.48.gate_proj", "model.layers.49.mlp.experts.49.gate_proj", "model.layers.49.mlp.experts.50.gate_proj", "model.layers.49.mlp.experts.51.gate_proj", "model.layers.49.mlp.experts.52.gate_proj", "model.layers.49.mlp.experts.53.gate_proj", "model.layers.49.mlp.experts.54.gate_proj", "model.layers.49.mlp.experts.55.gate_proj", "model.layers.49.mlp.experts.56.gate_proj", "model.layers.49.mlp.experts.57.gate_proj", "model.layers.49.mlp.experts.58.gate_proj", "model.layers.49.mlp.experts.59.gate_proj", "model.layers.49.mlp.experts.60.gate_proj", "model.layers.49.mlp.experts.61.gate_proj", "model.layers.49.mlp.experts.62.gate_proj", "model.layers.49.mlp.experts.63.gate_proj", "model.layers.49.mlp.experts.64.gate_proj", "model.layers.49.mlp.experts.65.gate_proj", "model.layers.49.mlp.experts.66.gate_proj", "model.layers.49.mlp.experts.67.gate_proj", "model.layers.49.mlp.experts.68.gate_proj", "model.layers.49.mlp.experts.69.gate_proj", "model.layers.49.mlp.experts.70.gate_proj", "model.layers.49.mlp.experts.71.gate_proj", "model.layers.49.mlp.experts.72.gate_proj", "model.layers.49.mlp.experts.73.gate_proj", "model.layers.49.mlp.experts.74.gate_proj", "model.layers.49.mlp.experts.75.gate_proj", "model.layers.49.mlp.experts.76.gate_proj", "model.layers.49.mlp.experts.77.gate_proj", "model.layers.49.mlp.experts.78.gate_proj", "model.layers.49.mlp.experts.79.gate_proj", "model.layers.49.mlp.experts.80.gate_proj", "model.layers.49.mlp.experts.81.gate_proj", "model.layers.49.mlp.experts.82.gate_proj", "model.layers.49.mlp.experts.83.gate_proj", "model.layers.49.mlp.experts.84.gate_proj", "model.layers.49.mlp.experts.85.gate_proj", "model.layers.49.mlp.experts.86.gate_proj", "model.layers.49.mlp.experts.87.gate_proj", "model.layers.49.mlp.experts.88.gate_proj", "model.layers.49.mlp.experts.89.gate_proj", "model.layers.49.mlp.experts.90.gate_proj", "model.layers.49.mlp.experts.91.gate_proj", "model.layers.49.mlp.experts.92.gate_proj", "model.layers.49.mlp.experts.93.gate_proj", "model.layers.49.mlp.experts.94.gate_proj", "model.layers.49.mlp.experts.95.gate_proj", "model.layers.49.mlp.experts.96.gate_proj", "model.layers.49.mlp.experts.97.gate_proj", "model.layers.49.mlp.experts.98.gate_proj", "model.layers.49.mlp.experts.99.gate_proj", "model.layers.49.mlp.experts.100.gate_proj", "model.layers.49.mlp.experts.101.gate_proj", "model.layers.49.mlp.experts.102.gate_proj", "model.layers.49.mlp.experts.103.gate_proj", "model.layers.49.mlp.experts.104.gate_proj", "model.layers.49.mlp.experts.105.gate_proj", "model.layers.49.mlp.experts.106.gate_proj", "model.layers.49.mlp.experts.107.gate_proj", "model.layers.49.mlp.experts.108.gate_proj", "model.layers.49.mlp.experts.109.gate_proj", "model.layers.49.mlp.experts.110.gate_proj", "model.layers.49.mlp.experts.111.gate_proj", "model.layers.49.mlp.experts.112.gate_proj", "model.layers.49.mlp.experts.113.gate_proj", "model.layers.49.mlp.experts.114.gate_proj", "model.layers.49.mlp.experts.115.gate_proj", "model.layers.49.mlp.experts.116.gate_proj", "model.layers.49.mlp.experts.117.gate_proj", "model.layers.49.mlp.experts.118.gate_proj", "model.layers.49.mlp.experts.119.gate_proj", "model.layers.49.mlp.experts.120.gate_proj", "model.layers.49.mlp.experts.121.gate_proj", "model.layers.49.mlp.experts.122.gate_proj", "model.layers.49.mlp.experts.123.gate_proj", "model.layers.49.mlp.experts.124.gate_proj", "model.layers.49.mlp.experts.125.gate_proj", "model.layers.49.mlp.experts.126.gate_proj", "model.layers.49.mlp.experts.127.gate_proj", "model.layers.49.mlp.experts.128.gate_proj", "model.layers.49.mlp.experts.129.gate_proj", "model.layers.49.mlp.experts.130.gate_proj", "model.layers.49.mlp.experts.131.gate_proj", "model.layers.49.mlp.experts.132.gate_proj", "model.layers.49.mlp.experts.133.gate_proj", "model.layers.49.mlp.experts.134.gate_proj", "model.layers.49.mlp.experts.135.gate_proj", "model.layers.49.mlp.experts.136.gate_proj", "model.layers.49.mlp.experts.137.gate_proj", "model.layers.49.mlp.experts.138.gate_proj", "model.layers.49.mlp.experts.139.gate_proj", "model.layers.49.mlp.experts.140.gate_proj", "model.layers.49.mlp.experts.141.gate_proj", "model.layers.49.mlp.experts.142.gate_proj", "model.layers.49.mlp.experts.143.gate_proj", "model.layers.49.mlp.experts.144.gate_proj", "model.layers.49.mlp.experts.145.gate_proj", "model.layers.49.mlp.experts.146.gate_proj", "model.layers.49.mlp.experts.147.gate_proj", "model.layers.49.mlp.experts.148.gate_proj", "model.layers.49.mlp.experts.149.gate_proj", "model.layers.49.mlp.experts.150.gate_proj", "model.layers.49.mlp.experts.151.gate_proj", "model.layers.49.mlp.experts.152.gate_proj", "model.layers.49.mlp.experts.153.gate_proj", "model.layers.49.mlp.experts.154.gate_proj", "model.layers.49.mlp.experts.155.gate_proj", "model.layers.49.mlp.experts.156.gate_proj", "model.layers.49.mlp.experts.157.gate_proj", "model.layers.49.mlp.experts.158.gate_proj", "model.layers.49.mlp.experts.159.gate_proj", "model.layers.49.mlp.experts.0.up_proj", "model.layers.49.mlp.experts.1.up_proj", "model.layers.49.mlp.experts.2.up_proj", "model.layers.49.mlp.experts.3.up_proj", "model.layers.49.mlp.experts.4.up_proj", "model.layers.49.mlp.experts.5.up_proj", "model.layers.49.mlp.experts.6.up_proj", "model.layers.49.mlp.experts.7.up_proj", "model.layers.49.mlp.experts.8.up_proj", "model.layers.49.mlp.experts.9.up_proj", "model.layers.49.mlp.experts.10.up_proj", "model.layers.49.mlp.experts.11.up_proj", "model.layers.49.mlp.experts.12.up_proj", "model.layers.49.mlp.experts.13.up_proj", "model.layers.49.mlp.experts.14.up_proj", "model.layers.49.mlp.experts.15.up_proj", "model.layers.49.mlp.experts.16.up_proj", "model.layers.49.mlp.experts.17.up_proj", "model.layers.49.mlp.experts.18.up_proj", "model.layers.49.mlp.experts.19.up_proj", "model.layers.49.mlp.experts.20.up_proj", "model.layers.49.mlp.experts.21.up_proj", "model.layers.49.mlp.experts.22.up_proj", "model.layers.49.mlp.experts.23.up_proj", "model.layers.49.mlp.experts.24.up_proj", "model.layers.49.mlp.experts.25.up_proj", "model.layers.49.mlp.experts.26.up_proj", "model.layers.49.mlp.experts.27.up_proj", "model.layers.49.mlp.experts.28.up_proj", "model.layers.49.mlp.experts.29.up_proj", "model.layers.49.mlp.experts.30.up_proj", "model.layers.49.mlp.experts.31.up_proj", "model.layers.49.mlp.experts.32.up_proj", "model.layers.49.mlp.experts.33.up_proj", "model.layers.49.mlp.experts.34.up_proj", "model.layers.49.mlp.experts.35.up_proj", "model.layers.49.mlp.experts.36.up_proj", "model.layers.49.mlp.experts.37.up_proj", "model.layers.49.mlp.experts.38.up_proj", "model.layers.49.mlp.experts.39.up_proj", "model.layers.49.mlp.experts.40.up_proj", "model.layers.49.mlp.experts.41.up_proj", "model.layers.49.mlp.experts.42.up_proj", "model.layers.49.mlp.experts.43.up_proj", "model.layers.49.mlp.experts.44.up_proj", "model.layers.49.mlp.experts.45.up_proj", "model.layers.49.mlp.experts.46.up_proj", "model.layers.49.mlp.experts.47.up_proj", "model.layers.49.mlp.experts.48.up_proj", "model.layers.49.mlp.experts.49.up_proj", "model.layers.49.mlp.experts.50.up_proj", "model.layers.49.mlp.experts.51.up_proj", "model.layers.49.mlp.experts.52.up_proj", "model.layers.49.mlp.experts.53.up_proj", "model.layers.49.mlp.experts.54.up_proj", "model.layers.49.mlp.experts.55.up_proj", "model.layers.49.mlp.experts.56.up_proj", "model.layers.49.mlp.experts.57.up_proj", "model.layers.49.mlp.experts.58.up_proj", "model.layers.49.mlp.experts.59.up_proj", "model.layers.49.mlp.experts.60.up_proj", "model.layers.49.mlp.experts.61.up_proj", "model.layers.49.mlp.experts.62.up_proj", "model.layers.49.mlp.experts.63.up_proj", "model.layers.49.mlp.experts.64.up_proj", "model.layers.49.mlp.experts.65.up_proj", "model.layers.49.mlp.experts.66.up_proj", "model.layers.49.mlp.experts.67.up_proj", "model.layers.49.mlp.experts.68.up_proj", "model.layers.49.mlp.experts.69.up_proj", "model.layers.49.mlp.experts.70.up_proj", "model.layers.49.mlp.experts.71.up_proj", "model.layers.49.mlp.experts.72.up_proj", "model.layers.49.mlp.experts.73.up_proj", "model.layers.49.mlp.experts.74.up_proj", "model.layers.49.mlp.experts.75.up_proj", "model.layers.49.mlp.experts.76.up_proj", "model.layers.49.mlp.experts.77.up_proj", "model.layers.49.mlp.experts.78.up_proj", "model.layers.49.mlp.experts.79.up_proj", "model.layers.49.mlp.experts.80.up_proj", "model.layers.49.mlp.experts.81.up_proj", "model.layers.49.mlp.experts.82.up_proj", "model.layers.49.mlp.experts.83.up_proj", "model.layers.49.mlp.experts.84.up_proj", "model.layers.49.mlp.experts.85.up_proj", "model.layers.49.mlp.experts.86.up_proj", "model.layers.49.mlp.experts.87.up_proj", "model.layers.49.mlp.experts.88.up_proj", "model.layers.49.mlp.experts.89.up_proj", "model.layers.49.mlp.experts.90.up_proj", "model.layers.49.mlp.experts.91.up_proj", "model.layers.49.mlp.experts.92.up_proj", "model.layers.49.mlp.experts.93.up_proj", "model.layers.49.mlp.experts.94.up_proj", "model.layers.49.mlp.experts.95.up_proj", "model.layers.49.mlp.experts.96.up_proj", "model.layers.49.mlp.experts.97.up_proj", "model.layers.49.mlp.experts.98.up_proj", "model.layers.49.mlp.experts.99.up_proj", "model.layers.49.mlp.experts.100.up_proj", "model.layers.49.mlp.experts.101.up_proj", "model.layers.49.mlp.experts.102.up_proj", "model.layers.49.mlp.experts.103.up_proj", "model.layers.49.mlp.experts.104.up_proj", "model.layers.49.mlp.experts.105.up_proj", "model.layers.49.mlp.experts.106.up_proj", "model.layers.49.mlp.experts.107.up_proj", "model.layers.49.mlp.experts.108.up_proj", "model.layers.49.mlp.experts.109.up_proj", "model.layers.49.mlp.experts.110.up_proj", "model.layers.49.mlp.experts.111.up_proj", "model.layers.49.mlp.experts.112.up_proj", "model.layers.49.mlp.experts.113.up_proj", "model.layers.49.mlp.experts.114.up_proj", "model.layers.49.mlp.experts.115.up_proj", "model.layers.49.mlp.experts.116.up_proj", "model.layers.49.mlp.experts.117.up_proj", "model.layers.49.mlp.experts.118.up_proj", "model.layers.49.mlp.experts.119.up_proj", "model.layers.49.mlp.experts.120.up_proj", "model.layers.49.mlp.experts.121.up_proj", "model.layers.49.mlp.experts.122.up_proj", "model.layers.49.mlp.experts.123.up_proj", "model.layers.49.mlp.experts.124.up_proj", "model.layers.49.mlp.experts.125.up_proj", "model.layers.49.mlp.experts.126.up_proj", "model.layers.49.mlp.experts.127.up_proj", "model.layers.49.mlp.experts.128.up_proj", "model.layers.49.mlp.experts.129.up_proj", "model.layers.49.mlp.experts.130.up_proj", "model.layers.49.mlp.experts.131.up_proj", "model.layers.49.mlp.experts.132.up_proj", "model.layers.49.mlp.experts.133.up_proj", "model.layers.49.mlp.experts.134.up_proj", "model.layers.49.mlp.experts.135.up_proj", "model.layers.49.mlp.experts.136.up_proj", "model.layers.49.mlp.experts.137.up_proj", "model.layers.49.mlp.experts.138.up_proj", "model.layers.49.mlp.experts.139.up_proj", "model.layers.49.mlp.experts.140.up_proj", "model.layers.49.mlp.experts.141.up_proj", "model.layers.49.mlp.experts.142.up_proj", "model.layers.49.mlp.experts.143.up_proj", "model.layers.49.mlp.experts.144.up_proj", "model.layers.49.mlp.experts.145.up_proj", "model.layers.49.mlp.experts.146.up_proj", "model.layers.49.mlp.experts.147.up_proj", "model.layers.49.mlp.experts.148.up_proj", "model.layers.49.mlp.experts.149.up_proj", "model.layers.49.mlp.experts.150.up_proj", "model.layers.49.mlp.experts.151.up_proj", "model.layers.49.mlp.experts.152.up_proj", "model.layers.49.mlp.experts.153.up_proj", "model.layers.49.mlp.experts.154.up_proj", "model.layers.49.mlp.experts.155.up_proj", "model.layers.49.mlp.experts.156.up_proj", "model.layers.49.mlp.experts.157.up_proj", "model.layers.49.mlp.experts.158.up_proj", "model.layers.49.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.702520138584121e-06, "dbits": 5033164800 } ] }, { "idx": 296, "layers": [ "model.layers.49.mlp.experts.0.down_proj", "model.layers.49.mlp.experts.1.down_proj", "model.layers.49.mlp.experts.2.down_proj", "model.layers.49.mlp.experts.3.down_proj", "model.layers.49.mlp.experts.4.down_proj", "model.layers.49.mlp.experts.5.down_proj", "model.layers.49.mlp.experts.6.down_proj", "model.layers.49.mlp.experts.7.down_proj", "model.layers.49.mlp.experts.8.down_proj", "model.layers.49.mlp.experts.9.down_proj", "model.layers.49.mlp.experts.10.down_proj", "model.layers.49.mlp.experts.11.down_proj", "model.layers.49.mlp.experts.12.down_proj", "model.layers.49.mlp.experts.13.down_proj", "model.layers.49.mlp.experts.14.down_proj", "model.layers.49.mlp.experts.15.down_proj", "model.layers.49.mlp.experts.16.down_proj", "model.layers.49.mlp.experts.17.down_proj", "model.layers.49.mlp.experts.18.down_proj", "model.layers.49.mlp.experts.19.down_proj", "model.layers.49.mlp.experts.20.down_proj", "model.layers.49.mlp.experts.21.down_proj", "model.layers.49.mlp.experts.22.down_proj", "model.layers.49.mlp.experts.23.down_proj", "model.layers.49.mlp.experts.24.down_proj", "model.layers.49.mlp.experts.25.down_proj", "model.layers.49.mlp.experts.26.down_proj", "model.layers.49.mlp.experts.27.down_proj", "model.layers.49.mlp.experts.28.down_proj", "model.layers.49.mlp.experts.29.down_proj", "model.layers.49.mlp.experts.30.down_proj", "model.layers.49.mlp.experts.31.down_proj", "model.layers.49.mlp.experts.32.down_proj", "model.layers.49.mlp.experts.33.down_proj", "model.layers.49.mlp.experts.34.down_proj", "model.layers.49.mlp.experts.35.down_proj", "model.layers.49.mlp.experts.36.down_proj", "model.layers.49.mlp.experts.37.down_proj", "model.layers.49.mlp.experts.38.down_proj", "model.layers.49.mlp.experts.39.down_proj", "model.layers.49.mlp.experts.40.down_proj", "model.layers.49.mlp.experts.41.down_proj", "model.layers.49.mlp.experts.42.down_proj", "model.layers.49.mlp.experts.43.down_proj", "model.layers.49.mlp.experts.44.down_proj", "model.layers.49.mlp.experts.45.down_proj", "model.layers.49.mlp.experts.46.down_proj", "model.layers.49.mlp.experts.47.down_proj", "model.layers.49.mlp.experts.48.down_proj", "model.layers.49.mlp.experts.49.down_proj", "model.layers.49.mlp.experts.50.down_proj", "model.layers.49.mlp.experts.51.down_proj", "model.layers.49.mlp.experts.52.down_proj", "model.layers.49.mlp.experts.53.down_proj", "model.layers.49.mlp.experts.54.down_proj", "model.layers.49.mlp.experts.55.down_proj", "model.layers.49.mlp.experts.56.down_proj", "model.layers.49.mlp.experts.57.down_proj", "model.layers.49.mlp.experts.58.down_proj", "model.layers.49.mlp.experts.59.down_proj", "model.layers.49.mlp.experts.60.down_proj", "model.layers.49.mlp.experts.61.down_proj", "model.layers.49.mlp.experts.62.down_proj", "model.layers.49.mlp.experts.63.down_proj", "model.layers.49.mlp.experts.64.down_proj", "model.layers.49.mlp.experts.65.down_proj", "model.layers.49.mlp.experts.66.down_proj", "model.layers.49.mlp.experts.67.down_proj", "model.layers.49.mlp.experts.68.down_proj", "model.layers.49.mlp.experts.69.down_proj", "model.layers.49.mlp.experts.70.down_proj", "model.layers.49.mlp.experts.71.down_proj", "model.layers.49.mlp.experts.72.down_proj", "model.layers.49.mlp.experts.73.down_proj", "model.layers.49.mlp.experts.74.down_proj", "model.layers.49.mlp.experts.75.down_proj", "model.layers.49.mlp.experts.76.down_proj", "model.layers.49.mlp.experts.77.down_proj", "model.layers.49.mlp.experts.78.down_proj", "model.layers.49.mlp.experts.79.down_proj", "model.layers.49.mlp.experts.80.down_proj", "model.layers.49.mlp.experts.81.down_proj", "model.layers.49.mlp.experts.82.down_proj", "model.layers.49.mlp.experts.83.down_proj", "model.layers.49.mlp.experts.84.down_proj", "model.layers.49.mlp.experts.85.down_proj", "model.layers.49.mlp.experts.86.down_proj", "model.layers.49.mlp.experts.87.down_proj", "model.layers.49.mlp.experts.88.down_proj", "model.layers.49.mlp.experts.89.down_proj", "model.layers.49.mlp.experts.90.down_proj", "model.layers.49.mlp.experts.91.down_proj", "model.layers.49.mlp.experts.92.down_proj", "model.layers.49.mlp.experts.93.down_proj", "model.layers.49.mlp.experts.94.down_proj", "model.layers.49.mlp.experts.95.down_proj", "model.layers.49.mlp.experts.96.down_proj", "model.layers.49.mlp.experts.97.down_proj", "model.layers.49.mlp.experts.98.down_proj", "model.layers.49.mlp.experts.99.down_proj", "model.layers.49.mlp.experts.100.down_proj", "model.layers.49.mlp.experts.101.down_proj", "model.layers.49.mlp.experts.102.down_proj", "model.layers.49.mlp.experts.103.down_proj", "model.layers.49.mlp.experts.104.down_proj", "model.layers.49.mlp.experts.105.down_proj", "model.layers.49.mlp.experts.106.down_proj", "model.layers.49.mlp.experts.107.down_proj", "model.layers.49.mlp.experts.108.down_proj", "model.layers.49.mlp.experts.109.down_proj", "model.layers.49.mlp.experts.110.down_proj", "model.layers.49.mlp.experts.111.down_proj", "model.layers.49.mlp.experts.112.down_proj", "model.layers.49.mlp.experts.113.down_proj", "model.layers.49.mlp.experts.114.down_proj", "model.layers.49.mlp.experts.115.down_proj", "model.layers.49.mlp.experts.116.down_proj", "model.layers.49.mlp.experts.117.down_proj", "model.layers.49.mlp.experts.118.down_proj", "model.layers.49.mlp.experts.119.down_proj", "model.layers.49.mlp.experts.120.down_proj", "model.layers.49.mlp.experts.121.down_proj", "model.layers.49.mlp.experts.122.down_proj", "model.layers.49.mlp.experts.123.down_proj", "model.layers.49.mlp.experts.124.down_proj", "model.layers.49.mlp.experts.125.down_proj", "model.layers.49.mlp.experts.126.down_proj", "model.layers.49.mlp.experts.127.down_proj", "model.layers.49.mlp.experts.128.down_proj", "model.layers.49.mlp.experts.129.down_proj", "model.layers.49.mlp.experts.130.down_proj", "model.layers.49.mlp.experts.131.down_proj", "model.layers.49.mlp.experts.132.down_proj", "model.layers.49.mlp.experts.133.down_proj", "model.layers.49.mlp.experts.134.down_proj", "model.layers.49.mlp.experts.135.down_proj", "model.layers.49.mlp.experts.136.down_proj", "model.layers.49.mlp.experts.137.down_proj", "model.layers.49.mlp.experts.138.down_proj", "model.layers.49.mlp.experts.139.down_proj", "model.layers.49.mlp.experts.140.down_proj", "model.layers.49.mlp.experts.141.down_proj", "model.layers.49.mlp.experts.142.down_proj", "model.layers.49.mlp.experts.143.down_proj", "model.layers.49.mlp.experts.144.down_proj", "model.layers.49.mlp.experts.145.down_proj", "model.layers.49.mlp.experts.146.down_proj", "model.layers.49.mlp.experts.147.down_proj", "model.layers.49.mlp.experts.148.down_proj", "model.layers.49.mlp.experts.149.down_proj", "model.layers.49.mlp.experts.150.down_proj", "model.layers.49.mlp.experts.151.down_proj", "model.layers.49.mlp.experts.152.down_proj", "model.layers.49.mlp.experts.153.down_proj", "model.layers.49.mlp.experts.154.down_proj", "model.layers.49.mlp.experts.155.down_proj", "model.layers.49.mlp.experts.156.down_proj", "model.layers.49.mlp.experts.157.down_proj", "model.layers.49.mlp.experts.158.down_proj", "model.layers.49.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -6.725225830450437e-07, "dbits": 2516582400 } ] }, { "idx": 297, "layers": [ "model.layers.50.self_attn.q_proj" ], "candidates": [ { "dkld": -8.378352504223802e-07, "dbits": 125829120 } ] }, { "idx": 298, "layers": [ "model.layers.50.self_attn.k_proj", "model.layers.50.self_attn.v_proj" ], "candidates": [ { "dkld": 6.747373845428445e-07, "dbits": 20971520 } ] }, { "idx": 299, "layers": [ "model.layers.50.self_attn.o_proj" ], "candidates": [ { "dkld": 1.1465890565887094e-06, "dbits": 125829120 } ] }, { "idx": 300, "layers": [ "model.layers.50.mlp.shared_experts.gate_proj", "model.layers.50.mlp.shared_experts.up_proj", "model.layers.50.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 5.7775207096711235e-06, "dbits": 47185920 } ] }, { "idx": 301, "layers": [ "model.layers.50.mlp.experts.0.gate_proj", "model.layers.50.mlp.experts.1.gate_proj", "model.layers.50.mlp.experts.2.gate_proj", "model.layers.50.mlp.experts.3.gate_proj", "model.layers.50.mlp.experts.4.gate_proj", "model.layers.50.mlp.experts.5.gate_proj", "model.layers.50.mlp.experts.6.gate_proj", "model.layers.50.mlp.experts.7.gate_proj", "model.layers.50.mlp.experts.8.gate_proj", "model.layers.50.mlp.experts.9.gate_proj", "model.layers.50.mlp.experts.10.gate_proj", "model.layers.50.mlp.experts.11.gate_proj", "model.layers.50.mlp.experts.12.gate_proj", "model.layers.50.mlp.experts.13.gate_proj", "model.layers.50.mlp.experts.14.gate_proj", "model.layers.50.mlp.experts.15.gate_proj", "model.layers.50.mlp.experts.16.gate_proj", "model.layers.50.mlp.experts.17.gate_proj", "model.layers.50.mlp.experts.18.gate_proj", "model.layers.50.mlp.experts.19.gate_proj", "model.layers.50.mlp.experts.20.gate_proj", "model.layers.50.mlp.experts.21.gate_proj", "model.layers.50.mlp.experts.22.gate_proj", "model.layers.50.mlp.experts.23.gate_proj", "model.layers.50.mlp.experts.24.gate_proj", "model.layers.50.mlp.experts.25.gate_proj", "model.layers.50.mlp.experts.26.gate_proj", "model.layers.50.mlp.experts.27.gate_proj", "model.layers.50.mlp.experts.28.gate_proj", "model.layers.50.mlp.experts.29.gate_proj", "model.layers.50.mlp.experts.30.gate_proj", "model.layers.50.mlp.experts.31.gate_proj", "model.layers.50.mlp.experts.32.gate_proj", "model.layers.50.mlp.experts.33.gate_proj", "model.layers.50.mlp.experts.34.gate_proj", "model.layers.50.mlp.experts.35.gate_proj", "model.layers.50.mlp.experts.36.gate_proj", "model.layers.50.mlp.experts.37.gate_proj", "model.layers.50.mlp.experts.38.gate_proj", "model.layers.50.mlp.experts.39.gate_proj", "model.layers.50.mlp.experts.40.gate_proj", "model.layers.50.mlp.experts.41.gate_proj", "model.layers.50.mlp.experts.42.gate_proj", "model.layers.50.mlp.experts.43.gate_proj", "model.layers.50.mlp.experts.44.gate_proj", "model.layers.50.mlp.experts.45.gate_proj", "model.layers.50.mlp.experts.46.gate_proj", "model.layers.50.mlp.experts.47.gate_proj", "model.layers.50.mlp.experts.48.gate_proj", "model.layers.50.mlp.experts.49.gate_proj", "model.layers.50.mlp.experts.50.gate_proj", "model.layers.50.mlp.experts.51.gate_proj", "model.layers.50.mlp.experts.52.gate_proj", "model.layers.50.mlp.experts.53.gate_proj", "model.layers.50.mlp.experts.54.gate_proj", "model.layers.50.mlp.experts.55.gate_proj", "model.layers.50.mlp.experts.56.gate_proj", "model.layers.50.mlp.experts.57.gate_proj", "model.layers.50.mlp.experts.58.gate_proj", "model.layers.50.mlp.experts.59.gate_proj", "model.layers.50.mlp.experts.60.gate_proj", "model.layers.50.mlp.experts.61.gate_proj", "model.layers.50.mlp.experts.62.gate_proj", "model.layers.50.mlp.experts.63.gate_proj", "model.layers.50.mlp.experts.64.gate_proj", "model.layers.50.mlp.experts.65.gate_proj", "model.layers.50.mlp.experts.66.gate_proj", "model.layers.50.mlp.experts.67.gate_proj", "model.layers.50.mlp.experts.68.gate_proj", "model.layers.50.mlp.experts.69.gate_proj", "model.layers.50.mlp.experts.70.gate_proj", "model.layers.50.mlp.experts.71.gate_proj", "model.layers.50.mlp.experts.72.gate_proj", "model.layers.50.mlp.experts.73.gate_proj", "model.layers.50.mlp.experts.74.gate_proj", "model.layers.50.mlp.experts.75.gate_proj", "model.layers.50.mlp.experts.76.gate_proj", "model.layers.50.mlp.experts.77.gate_proj", "model.layers.50.mlp.experts.78.gate_proj", "model.layers.50.mlp.experts.79.gate_proj", "model.layers.50.mlp.experts.80.gate_proj", "model.layers.50.mlp.experts.81.gate_proj", "model.layers.50.mlp.experts.82.gate_proj", "model.layers.50.mlp.experts.83.gate_proj", "model.layers.50.mlp.experts.84.gate_proj", "model.layers.50.mlp.experts.85.gate_proj", "model.layers.50.mlp.experts.86.gate_proj", "model.layers.50.mlp.experts.87.gate_proj", "model.layers.50.mlp.experts.88.gate_proj", "model.layers.50.mlp.experts.89.gate_proj", "model.layers.50.mlp.experts.90.gate_proj", "model.layers.50.mlp.experts.91.gate_proj", "model.layers.50.mlp.experts.92.gate_proj", "model.layers.50.mlp.experts.93.gate_proj", "model.layers.50.mlp.experts.94.gate_proj", "model.layers.50.mlp.experts.95.gate_proj", "model.layers.50.mlp.experts.96.gate_proj", "model.layers.50.mlp.experts.97.gate_proj", "model.layers.50.mlp.experts.98.gate_proj", "model.layers.50.mlp.experts.99.gate_proj", "model.layers.50.mlp.experts.100.gate_proj", "model.layers.50.mlp.experts.101.gate_proj", "model.layers.50.mlp.experts.102.gate_proj", "model.layers.50.mlp.experts.103.gate_proj", "model.layers.50.mlp.experts.104.gate_proj", "model.layers.50.mlp.experts.105.gate_proj", "model.layers.50.mlp.experts.106.gate_proj", "model.layers.50.mlp.experts.107.gate_proj", "model.layers.50.mlp.experts.108.gate_proj", "model.layers.50.mlp.experts.109.gate_proj", "model.layers.50.mlp.experts.110.gate_proj", "model.layers.50.mlp.experts.111.gate_proj", "model.layers.50.mlp.experts.112.gate_proj", "model.layers.50.mlp.experts.113.gate_proj", "model.layers.50.mlp.experts.114.gate_proj", "model.layers.50.mlp.experts.115.gate_proj", "model.layers.50.mlp.experts.116.gate_proj", "model.layers.50.mlp.experts.117.gate_proj", "model.layers.50.mlp.experts.118.gate_proj", "model.layers.50.mlp.experts.119.gate_proj", "model.layers.50.mlp.experts.120.gate_proj", "model.layers.50.mlp.experts.121.gate_proj", "model.layers.50.mlp.experts.122.gate_proj", "model.layers.50.mlp.experts.123.gate_proj", "model.layers.50.mlp.experts.124.gate_proj", "model.layers.50.mlp.experts.125.gate_proj", "model.layers.50.mlp.experts.126.gate_proj", "model.layers.50.mlp.experts.127.gate_proj", "model.layers.50.mlp.experts.128.gate_proj", "model.layers.50.mlp.experts.129.gate_proj", "model.layers.50.mlp.experts.130.gate_proj", "model.layers.50.mlp.experts.131.gate_proj", "model.layers.50.mlp.experts.132.gate_proj", "model.layers.50.mlp.experts.133.gate_proj", "model.layers.50.mlp.experts.134.gate_proj", "model.layers.50.mlp.experts.135.gate_proj", "model.layers.50.mlp.experts.136.gate_proj", "model.layers.50.mlp.experts.137.gate_proj", "model.layers.50.mlp.experts.138.gate_proj", "model.layers.50.mlp.experts.139.gate_proj", "model.layers.50.mlp.experts.140.gate_proj", "model.layers.50.mlp.experts.141.gate_proj", "model.layers.50.mlp.experts.142.gate_proj", "model.layers.50.mlp.experts.143.gate_proj", "model.layers.50.mlp.experts.144.gate_proj", "model.layers.50.mlp.experts.145.gate_proj", "model.layers.50.mlp.experts.146.gate_proj", "model.layers.50.mlp.experts.147.gate_proj", "model.layers.50.mlp.experts.148.gate_proj", "model.layers.50.mlp.experts.149.gate_proj", "model.layers.50.mlp.experts.150.gate_proj", "model.layers.50.mlp.experts.151.gate_proj", "model.layers.50.mlp.experts.152.gate_proj", "model.layers.50.mlp.experts.153.gate_proj", "model.layers.50.mlp.experts.154.gate_proj", "model.layers.50.mlp.experts.155.gate_proj", "model.layers.50.mlp.experts.156.gate_proj", "model.layers.50.mlp.experts.157.gate_proj", "model.layers.50.mlp.experts.158.gate_proj", "model.layers.50.mlp.experts.159.gate_proj", "model.layers.50.mlp.experts.0.up_proj", "model.layers.50.mlp.experts.1.up_proj", "model.layers.50.mlp.experts.2.up_proj", "model.layers.50.mlp.experts.3.up_proj", "model.layers.50.mlp.experts.4.up_proj", "model.layers.50.mlp.experts.5.up_proj", "model.layers.50.mlp.experts.6.up_proj", "model.layers.50.mlp.experts.7.up_proj", "model.layers.50.mlp.experts.8.up_proj", "model.layers.50.mlp.experts.9.up_proj", "model.layers.50.mlp.experts.10.up_proj", "model.layers.50.mlp.experts.11.up_proj", "model.layers.50.mlp.experts.12.up_proj", "model.layers.50.mlp.experts.13.up_proj", "model.layers.50.mlp.experts.14.up_proj", "model.layers.50.mlp.experts.15.up_proj", "model.layers.50.mlp.experts.16.up_proj", "model.layers.50.mlp.experts.17.up_proj", "model.layers.50.mlp.experts.18.up_proj", "model.layers.50.mlp.experts.19.up_proj", "model.layers.50.mlp.experts.20.up_proj", "model.layers.50.mlp.experts.21.up_proj", "model.layers.50.mlp.experts.22.up_proj", "model.layers.50.mlp.experts.23.up_proj", "model.layers.50.mlp.experts.24.up_proj", "model.layers.50.mlp.experts.25.up_proj", "model.layers.50.mlp.experts.26.up_proj", "model.layers.50.mlp.experts.27.up_proj", "model.layers.50.mlp.experts.28.up_proj", "model.layers.50.mlp.experts.29.up_proj", "model.layers.50.mlp.experts.30.up_proj", "model.layers.50.mlp.experts.31.up_proj", "model.layers.50.mlp.experts.32.up_proj", "model.layers.50.mlp.experts.33.up_proj", "model.layers.50.mlp.experts.34.up_proj", "model.layers.50.mlp.experts.35.up_proj", "model.layers.50.mlp.experts.36.up_proj", "model.layers.50.mlp.experts.37.up_proj", "model.layers.50.mlp.experts.38.up_proj", "model.layers.50.mlp.experts.39.up_proj", "model.layers.50.mlp.experts.40.up_proj", "model.layers.50.mlp.experts.41.up_proj", "model.layers.50.mlp.experts.42.up_proj", "model.layers.50.mlp.experts.43.up_proj", "model.layers.50.mlp.experts.44.up_proj", "model.layers.50.mlp.experts.45.up_proj", "model.layers.50.mlp.experts.46.up_proj", "model.layers.50.mlp.experts.47.up_proj", "model.layers.50.mlp.experts.48.up_proj", "model.layers.50.mlp.experts.49.up_proj", "model.layers.50.mlp.experts.50.up_proj", "model.layers.50.mlp.experts.51.up_proj", "model.layers.50.mlp.experts.52.up_proj", "model.layers.50.mlp.experts.53.up_proj", "model.layers.50.mlp.experts.54.up_proj", "model.layers.50.mlp.experts.55.up_proj", "model.layers.50.mlp.experts.56.up_proj", "model.layers.50.mlp.experts.57.up_proj", "model.layers.50.mlp.experts.58.up_proj", "model.layers.50.mlp.experts.59.up_proj", "model.layers.50.mlp.experts.60.up_proj", "model.layers.50.mlp.experts.61.up_proj", "model.layers.50.mlp.experts.62.up_proj", "model.layers.50.mlp.experts.63.up_proj", "model.layers.50.mlp.experts.64.up_proj", "model.layers.50.mlp.experts.65.up_proj", "model.layers.50.mlp.experts.66.up_proj", "model.layers.50.mlp.experts.67.up_proj", "model.layers.50.mlp.experts.68.up_proj", "model.layers.50.mlp.experts.69.up_proj", "model.layers.50.mlp.experts.70.up_proj", "model.layers.50.mlp.experts.71.up_proj", "model.layers.50.mlp.experts.72.up_proj", "model.layers.50.mlp.experts.73.up_proj", "model.layers.50.mlp.experts.74.up_proj", "model.layers.50.mlp.experts.75.up_proj", "model.layers.50.mlp.experts.76.up_proj", "model.layers.50.mlp.experts.77.up_proj", "model.layers.50.mlp.experts.78.up_proj", "model.layers.50.mlp.experts.79.up_proj", "model.layers.50.mlp.experts.80.up_proj", "model.layers.50.mlp.experts.81.up_proj", "model.layers.50.mlp.experts.82.up_proj", "model.layers.50.mlp.experts.83.up_proj", "model.layers.50.mlp.experts.84.up_proj", "model.layers.50.mlp.experts.85.up_proj", "model.layers.50.mlp.experts.86.up_proj", "model.layers.50.mlp.experts.87.up_proj", "model.layers.50.mlp.experts.88.up_proj", "model.layers.50.mlp.experts.89.up_proj", "model.layers.50.mlp.experts.90.up_proj", "model.layers.50.mlp.experts.91.up_proj", "model.layers.50.mlp.experts.92.up_proj", "model.layers.50.mlp.experts.93.up_proj", "model.layers.50.mlp.experts.94.up_proj", "model.layers.50.mlp.experts.95.up_proj", "model.layers.50.mlp.experts.96.up_proj", "model.layers.50.mlp.experts.97.up_proj", "model.layers.50.mlp.experts.98.up_proj", "model.layers.50.mlp.experts.99.up_proj", "model.layers.50.mlp.experts.100.up_proj", "model.layers.50.mlp.experts.101.up_proj", "model.layers.50.mlp.experts.102.up_proj", "model.layers.50.mlp.experts.103.up_proj", "model.layers.50.mlp.experts.104.up_proj", "model.layers.50.mlp.experts.105.up_proj", "model.layers.50.mlp.experts.106.up_proj", "model.layers.50.mlp.experts.107.up_proj", "model.layers.50.mlp.experts.108.up_proj", "model.layers.50.mlp.experts.109.up_proj", "model.layers.50.mlp.experts.110.up_proj", "model.layers.50.mlp.experts.111.up_proj", "model.layers.50.mlp.experts.112.up_proj", "model.layers.50.mlp.experts.113.up_proj", "model.layers.50.mlp.experts.114.up_proj", "model.layers.50.mlp.experts.115.up_proj", "model.layers.50.mlp.experts.116.up_proj", "model.layers.50.mlp.experts.117.up_proj", "model.layers.50.mlp.experts.118.up_proj", "model.layers.50.mlp.experts.119.up_proj", "model.layers.50.mlp.experts.120.up_proj", "model.layers.50.mlp.experts.121.up_proj", "model.layers.50.mlp.experts.122.up_proj", "model.layers.50.mlp.experts.123.up_proj", "model.layers.50.mlp.experts.124.up_proj", "model.layers.50.mlp.experts.125.up_proj", "model.layers.50.mlp.experts.126.up_proj", "model.layers.50.mlp.experts.127.up_proj", "model.layers.50.mlp.experts.128.up_proj", "model.layers.50.mlp.experts.129.up_proj", "model.layers.50.mlp.experts.130.up_proj", "model.layers.50.mlp.experts.131.up_proj", "model.layers.50.mlp.experts.132.up_proj", "model.layers.50.mlp.experts.133.up_proj", "model.layers.50.mlp.experts.134.up_proj", "model.layers.50.mlp.experts.135.up_proj", "model.layers.50.mlp.experts.136.up_proj", "model.layers.50.mlp.experts.137.up_proj", "model.layers.50.mlp.experts.138.up_proj", "model.layers.50.mlp.experts.139.up_proj", "model.layers.50.mlp.experts.140.up_proj", "model.layers.50.mlp.experts.141.up_proj", "model.layers.50.mlp.experts.142.up_proj", "model.layers.50.mlp.experts.143.up_proj", "model.layers.50.mlp.experts.144.up_proj", "model.layers.50.mlp.experts.145.up_proj", "model.layers.50.mlp.experts.146.up_proj", "model.layers.50.mlp.experts.147.up_proj", "model.layers.50.mlp.experts.148.up_proj", "model.layers.50.mlp.experts.149.up_proj", "model.layers.50.mlp.experts.150.up_proj", "model.layers.50.mlp.experts.151.up_proj", "model.layers.50.mlp.experts.152.up_proj", "model.layers.50.mlp.experts.153.up_proj", "model.layers.50.mlp.experts.154.up_proj", "model.layers.50.mlp.experts.155.up_proj", "model.layers.50.mlp.experts.156.up_proj", "model.layers.50.mlp.experts.157.up_proj", "model.layers.50.mlp.experts.158.up_proj", "model.layers.50.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -1.087272539734797e-06, "dbits": 5033164800 } ] }, { "idx": 302, "layers": [ "model.layers.50.mlp.experts.0.down_proj", "model.layers.50.mlp.experts.1.down_proj", "model.layers.50.mlp.experts.2.down_proj", "model.layers.50.mlp.experts.3.down_proj", "model.layers.50.mlp.experts.4.down_proj", "model.layers.50.mlp.experts.5.down_proj", "model.layers.50.mlp.experts.6.down_proj", "model.layers.50.mlp.experts.7.down_proj", "model.layers.50.mlp.experts.8.down_proj", "model.layers.50.mlp.experts.9.down_proj", "model.layers.50.mlp.experts.10.down_proj", "model.layers.50.mlp.experts.11.down_proj", "model.layers.50.mlp.experts.12.down_proj", "model.layers.50.mlp.experts.13.down_proj", "model.layers.50.mlp.experts.14.down_proj", "model.layers.50.mlp.experts.15.down_proj", "model.layers.50.mlp.experts.16.down_proj", "model.layers.50.mlp.experts.17.down_proj", "model.layers.50.mlp.experts.18.down_proj", "model.layers.50.mlp.experts.19.down_proj", "model.layers.50.mlp.experts.20.down_proj", "model.layers.50.mlp.experts.21.down_proj", "model.layers.50.mlp.experts.22.down_proj", "model.layers.50.mlp.experts.23.down_proj", "model.layers.50.mlp.experts.24.down_proj", "model.layers.50.mlp.experts.25.down_proj", "model.layers.50.mlp.experts.26.down_proj", "model.layers.50.mlp.experts.27.down_proj", "model.layers.50.mlp.experts.28.down_proj", "model.layers.50.mlp.experts.29.down_proj", "model.layers.50.mlp.experts.30.down_proj", "model.layers.50.mlp.experts.31.down_proj", "model.layers.50.mlp.experts.32.down_proj", "model.layers.50.mlp.experts.33.down_proj", "model.layers.50.mlp.experts.34.down_proj", "model.layers.50.mlp.experts.35.down_proj", "model.layers.50.mlp.experts.36.down_proj", "model.layers.50.mlp.experts.37.down_proj", "model.layers.50.mlp.experts.38.down_proj", "model.layers.50.mlp.experts.39.down_proj", "model.layers.50.mlp.experts.40.down_proj", "model.layers.50.mlp.experts.41.down_proj", "model.layers.50.mlp.experts.42.down_proj", "model.layers.50.mlp.experts.43.down_proj", "model.layers.50.mlp.experts.44.down_proj", "model.layers.50.mlp.experts.45.down_proj", "model.layers.50.mlp.experts.46.down_proj", "model.layers.50.mlp.experts.47.down_proj", "model.layers.50.mlp.experts.48.down_proj", "model.layers.50.mlp.experts.49.down_proj", "model.layers.50.mlp.experts.50.down_proj", "model.layers.50.mlp.experts.51.down_proj", "model.layers.50.mlp.experts.52.down_proj", "model.layers.50.mlp.experts.53.down_proj", "model.layers.50.mlp.experts.54.down_proj", "model.layers.50.mlp.experts.55.down_proj", "model.layers.50.mlp.experts.56.down_proj", "model.layers.50.mlp.experts.57.down_proj", "model.layers.50.mlp.experts.58.down_proj", "model.layers.50.mlp.experts.59.down_proj", "model.layers.50.mlp.experts.60.down_proj", "model.layers.50.mlp.experts.61.down_proj", "model.layers.50.mlp.experts.62.down_proj", "model.layers.50.mlp.experts.63.down_proj", "model.layers.50.mlp.experts.64.down_proj", "model.layers.50.mlp.experts.65.down_proj", "model.layers.50.mlp.experts.66.down_proj", "model.layers.50.mlp.experts.67.down_proj", "model.layers.50.mlp.experts.68.down_proj", "model.layers.50.mlp.experts.69.down_proj", "model.layers.50.mlp.experts.70.down_proj", "model.layers.50.mlp.experts.71.down_proj", "model.layers.50.mlp.experts.72.down_proj", "model.layers.50.mlp.experts.73.down_proj", "model.layers.50.mlp.experts.74.down_proj", "model.layers.50.mlp.experts.75.down_proj", "model.layers.50.mlp.experts.76.down_proj", "model.layers.50.mlp.experts.77.down_proj", "model.layers.50.mlp.experts.78.down_proj", "model.layers.50.mlp.experts.79.down_proj", "model.layers.50.mlp.experts.80.down_proj", "model.layers.50.mlp.experts.81.down_proj", "model.layers.50.mlp.experts.82.down_proj", "model.layers.50.mlp.experts.83.down_proj", "model.layers.50.mlp.experts.84.down_proj", "model.layers.50.mlp.experts.85.down_proj", "model.layers.50.mlp.experts.86.down_proj", "model.layers.50.mlp.experts.87.down_proj", "model.layers.50.mlp.experts.88.down_proj", "model.layers.50.mlp.experts.89.down_proj", "model.layers.50.mlp.experts.90.down_proj", "model.layers.50.mlp.experts.91.down_proj", "model.layers.50.mlp.experts.92.down_proj", "model.layers.50.mlp.experts.93.down_proj", "model.layers.50.mlp.experts.94.down_proj", "model.layers.50.mlp.experts.95.down_proj", "model.layers.50.mlp.experts.96.down_proj", "model.layers.50.mlp.experts.97.down_proj", "model.layers.50.mlp.experts.98.down_proj", "model.layers.50.mlp.experts.99.down_proj", "model.layers.50.mlp.experts.100.down_proj", "model.layers.50.mlp.experts.101.down_proj", "model.layers.50.mlp.experts.102.down_proj", "model.layers.50.mlp.experts.103.down_proj", "model.layers.50.mlp.experts.104.down_proj", "model.layers.50.mlp.experts.105.down_proj", "model.layers.50.mlp.experts.106.down_proj", "model.layers.50.mlp.experts.107.down_proj", "model.layers.50.mlp.experts.108.down_proj", "model.layers.50.mlp.experts.109.down_proj", "model.layers.50.mlp.experts.110.down_proj", "model.layers.50.mlp.experts.111.down_proj", "model.layers.50.mlp.experts.112.down_proj", "model.layers.50.mlp.experts.113.down_proj", "model.layers.50.mlp.experts.114.down_proj", "model.layers.50.mlp.experts.115.down_proj", "model.layers.50.mlp.experts.116.down_proj", "model.layers.50.mlp.experts.117.down_proj", "model.layers.50.mlp.experts.118.down_proj", "model.layers.50.mlp.experts.119.down_proj", "model.layers.50.mlp.experts.120.down_proj", "model.layers.50.mlp.experts.121.down_proj", "model.layers.50.mlp.experts.122.down_proj", "model.layers.50.mlp.experts.123.down_proj", "model.layers.50.mlp.experts.124.down_proj", "model.layers.50.mlp.experts.125.down_proj", "model.layers.50.mlp.experts.126.down_proj", "model.layers.50.mlp.experts.127.down_proj", "model.layers.50.mlp.experts.128.down_proj", "model.layers.50.mlp.experts.129.down_proj", "model.layers.50.mlp.experts.130.down_proj", "model.layers.50.mlp.experts.131.down_proj", "model.layers.50.mlp.experts.132.down_proj", "model.layers.50.mlp.experts.133.down_proj", "model.layers.50.mlp.experts.134.down_proj", "model.layers.50.mlp.experts.135.down_proj", "model.layers.50.mlp.experts.136.down_proj", "model.layers.50.mlp.experts.137.down_proj", "model.layers.50.mlp.experts.138.down_proj", "model.layers.50.mlp.experts.139.down_proj", "model.layers.50.mlp.experts.140.down_proj", "model.layers.50.mlp.experts.141.down_proj", "model.layers.50.mlp.experts.142.down_proj", "model.layers.50.mlp.experts.143.down_proj", "model.layers.50.mlp.experts.144.down_proj", "model.layers.50.mlp.experts.145.down_proj", "model.layers.50.mlp.experts.146.down_proj", "model.layers.50.mlp.experts.147.down_proj", "model.layers.50.mlp.experts.148.down_proj", "model.layers.50.mlp.experts.149.down_proj", "model.layers.50.mlp.experts.150.down_proj", "model.layers.50.mlp.experts.151.down_proj", "model.layers.50.mlp.experts.152.down_proj", "model.layers.50.mlp.experts.153.down_proj", "model.layers.50.mlp.experts.154.down_proj", "model.layers.50.mlp.experts.155.down_proj", "model.layers.50.mlp.experts.156.down_proj", "model.layers.50.mlp.experts.157.down_proj", "model.layers.50.mlp.experts.158.down_proj", "model.layers.50.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.800210495479405e-06, "dbits": 2516582400 } ] }, { "idx": 303, "layers": [ "model.layers.51.self_attn.q_proj" ], "candidates": [ { "dkld": 2.330134157091401e-06, "dbits": 125829120 } ] }, { "idx": 304, "layers": [ "model.layers.51.self_attn.k_proj", "model.layers.51.self_attn.v_proj" ], "candidates": [ { "dkld": 3.3419666578993635e-06, "dbits": 20971520 } ] }, { "idx": 305, "layers": [ "model.layers.51.self_attn.o_proj" ], "candidates": [ { "dkld": -8.331204298883893e-07, "dbits": 125829120 } ] }, { "idx": 306, "layers": [ "model.layers.51.mlp.shared_experts.gate_proj", "model.layers.51.mlp.shared_experts.up_proj", "model.layers.51.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.8080132324248986e-06, "dbits": 47185920 } ] }, { "idx": 307, "layers": [ "model.layers.51.mlp.experts.0.gate_proj", "model.layers.51.mlp.experts.1.gate_proj", "model.layers.51.mlp.experts.2.gate_proj", "model.layers.51.mlp.experts.3.gate_proj", "model.layers.51.mlp.experts.4.gate_proj", "model.layers.51.mlp.experts.5.gate_proj", "model.layers.51.mlp.experts.6.gate_proj", "model.layers.51.mlp.experts.7.gate_proj", "model.layers.51.mlp.experts.8.gate_proj", "model.layers.51.mlp.experts.9.gate_proj", "model.layers.51.mlp.experts.10.gate_proj", "model.layers.51.mlp.experts.11.gate_proj", "model.layers.51.mlp.experts.12.gate_proj", "model.layers.51.mlp.experts.13.gate_proj", "model.layers.51.mlp.experts.14.gate_proj", "model.layers.51.mlp.experts.15.gate_proj", "model.layers.51.mlp.experts.16.gate_proj", "model.layers.51.mlp.experts.17.gate_proj", "model.layers.51.mlp.experts.18.gate_proj", "model.layers.51.mlp.experts.19.gate_proj", "model.layers.51.mlp.experts.20.gate_proj", "model.layers.51.mlp.experts.21.gate_proj", "model.layers.51.mlp.experts.22.gate_proj", "model.layers.51.mlp.experts.23.gate_proj", "model.layers.51.mlp.experts.24.gate_proj", "model.layers.51.mlp.experts.25.gate_proj", "model.layers.51.mlp.experts.26.gate_proj", "model.layers.51.mlp.experts.27.gate_proj", "model.layers.51.mlp.experts.28.gate_proj", "model.layers.51.mlp.experts.29.gate_proj", "model.layers.51.mlp.experts.30.gate_proj", "model.layers.51.mlp.experts.31.gate_proj", "model.layers.51.mlp.experts.32.gate_proj", "model.layers.51.mlp.experts.33.gate_proj", "model.layers.51.mlp.experts.34.gate_proj", "model.layers.51.mlp.experts.35.gate_proj", "model.layers.51.mlp.experts.36.gate_proj", "model.layers.51.mlp.experts.37.gate_proj", "model.layers.51.mlp.experts.38.gate_proj", "model.layers.51.mlp.experts.39.gate_proj", "model.layers.51.mlp.experts.40.gate_proj", "model.layers.51.mlp.experts.41.gate_proj", "model.layers.51.mlp.experts.42.gate_proj", "model.layers.51.mlp.experts.43.gate_proj", "model.layers.51.mlp.experts.44.gate_proj", "model.layers.51.mlp.experts.45.gate_proj", "model.layers.51.mlp.experts.46.gate_proj", "model.layers.51.mlp.experts.47.gate_proj", "model.layers.51.mlp.experts.48.gate_proj", "model.layers.51.mlp.experts.49.gate_proj", "model.layers.51.mlp.experts.50.gate_proj", "model.layers.51.mlp.experts.51.gate_proj", "model.layers.51.mlp.experts.52.gate_proj", "model.layers.51.mlp.experts.53.gate_proj", "model.layers.51.mlp.experts.54.gate_proj", "model.layers.51.mlp.experts.55.gate_proj", "model.layers.51.mlp.experts.56.gate_proj", "model.layers.51.mlp.experts.57.gate_proj", "model.layers.51.mlp.experts.58.gate_proj", "model.layers.51.mlp.experts.59.gate_proj", "model.layers.51.mlp.experts.60.gate_proj", "model.layers.51.mlp.experts.61.gate_proj", "model.layers.51.mlp.experts.62.gate_proj", "model.layers.51.mlp.experts.63.gate_proj", "model.layers.51.mlp.experts.64.gate_proj", "model.layers.51.mlp.experts.65.gate_proj", "model.layers.51.mlp.experts.66.gate_proj", "model.layers.51.mlp.experts.67.gate_proj", "model.layers.51.mlp.experts.68.gate_proj", "model.layers.51.mlp.experts.69.gate_proj", "model.layers.51.mlp.experts.70.gate_proj", "model.layers.51.mlp.experts.71.gate_proj", "model.layers.51.mlp.experts.72.gate_proj", "model.layers.51.mlp.experts.73.gate_proj", "model.layers.51.mlp.experts.74.gate_proj", "model.layers.51.mlp.experts.75.gate_proj", "model.layers.51.mlp.experts.76.gate_proj", "model.layers.51.mlp.experts.77.gate_proj", "model.layers.51.mlp.experts.78.gate_proj", "model.layers.51.mlp.experts.79.gate_proj", "model.layers.51.mlp.experts.80.gate_proj", "model.layers.51.mlp.experts.81.gate_proj", "model.layers.51.mlp.experts.82.gate_proj", "model.layers.51.mlp.experts.83.gate_proj", "model.layers.51.mlp.experts.84.gate_proj", "model.layers.51.mlp.experts.85.gate_proj", "model.layers.51.mlp.experts.86.gate_proj", "model.layers.51.mlp.experts.87.gate_proj", "model.layers.51.mlp.experts.88.gate_proj", "model.layers.51.mlp.experts.89.gate_proj", "model.layers.51.mlp.experts.90.gate_proj", "model.layers.51.mlp.experts.91.gate_proj", "model.layers.51.mlp.experts.92.gate_proj", "model.layers.51.mlp.experts.93.gate_proj", "model.layers.51.mlp.experts.94.gate_proj", "model.layers.51.mlp.experts.95.gate_proj", "model.layers.51.mlp.experts.96.gate_proj", "model.layers.51.mlp.experts.97.gate_proj", "model.layers.51.mlp.experts.98.gate_proj", "model.layers.51.mlp.experts.99.gate_proj", "model.layers.51.mlp.experts.100.gate_proj", "model.layers.51.mlp.experts.101.gate_proj", "model.layers.51.mlp.experts.102.gate_proj", "model.layers.51.mlp.experts.103.gate_proj", "model.layers.51.mlp.experts.104.gate_proj", "model.layers.51.mlp.experts.105.gate_proj", "model.layers.51.mlp.experts.106.gate_proj", "model.layers.51.mlp.experts.107.gate_proj", "model.layers.51.mlp.experts.108.gate_proj", "model.layers.51.mlp.experts.109.gate_proj", "model.layers.51.mlp.experts.110.gate_proj", "model.layers.51.mlp.experts.111.gate_proj", "model.layers.51.mlp.experts.112.gate_proj", "model.layers.51.mlp.experts.113.gate_proj", "model.layers.51.mlp.experts.114.gate_proj", "model.layers.51.mlp.experts.115.gate_proj", "model.layers.51.mlp.experts.116.gate_proj", "model.layers.51.mlp.experts.117.gate_proj", "model.layers.51.mlp.experts.118.gate_proj", "model.layers.51.mlp.experts.119.gate_proj", "model.layers.51.mlp.experts.120.gate_proj", "model.layers.51.mlp.experts.121.gate_proj", "model.layers.51.mlp.experts.122.gate_proj", "model.layers.51.mlp.experts.123.gate_proj", "model.layers.51.mlp.experts.124.gate_proj", "model.layers.51.mlp.experts.125.gate_proj", "model.layers.51.mlp.experts.126.gate_proj", "model.layers.51.mlp.experts.127.gate_proj", "model.layers.51.mlp.experts.128.gate_proj", "model.layers.51.mlp.experts.129.gate_proj", "model.layers.51.mlp.experts.130.gate_proj", "model.layers.51.mlp.experts.131.gate_proj", "model.layers.51.mlp.experts.132.gate_proj", "model.layers.51.mlp.experts.133.gate_proj", "model.layers.51.mlp.experts.134.gate_proj", "model.layers.51.mlp.experts.135.gate_proj", "model.layers.51.mlp.experts.136.gate_proj", "model.layers.51.mlp.experts.137.gate_proj", "model.layers.51.mlp.experts.138.gate_proj", "model.layers.51.mlp.experts.139.gate_proj", "model.layers.51.mlp.experts.140.gate_proj", "model.layers.51.mlp.experts.141.gate_proj", "model.layers.51.mlp.experts.142.gate_proj", "model.layers.51.mlp.experts.143.gate_proj", "model.layers.51.mlp.experts.144.gate_proj", "model.layers.51.mlp.experts.145.gate_proj", "model.layers.51.mlp.experts.146.gate_proj", "model.layers.51.mlp.experts.147.gate_proj", "model.layers.51.mlp.experts.148.gate_proj", "model.layers.51.mlp.experts.149.gate_proj", "model.layers.51.mlp.experts.150.gate_proj", "model.layers.51.mlp.experts.151.gate_proj", "model.layers.51.mlp.experts.152.gate_proj", "model.layers.51.mlp.experts.153.gate_proj", "model.layers.51.mlp.experts.154.gate_proj", "model.layers.51.mlp.experts.155.gate_proj", "model.layers.51.mlp.experts.156.gate_proj", "model.layers.51.mlp.experts.157.gate_proj", "model.layers.51.mlp.experts.158.gate_proj", "model.layers.51.mlp.experts.159.gate_proj", "model.layers.51.mlp.experts.0.up_proj", "model.layers.51.mlp.experts.1.up_proj", "model.layers.51.mlp.experts.2.up_proj", "model.layers.51.mlp.experts.3.up_proj", "model.layers.51.mlp.experts.4.up_proj", "model.layers.51.mlp.experts.5.up_proj", "model.layers.51.mlp.experts.6.up_proj", "model.layers.51.mlp.experts.7.up_proj", "model.layers.51.mlp.experts.8.up_proj", "model.layers.51.mlp.experts.9.up_proj", "model.layers.51.mlp.experts.10.up_proj", "model.layers.51.mlp.experts.11.up_proj", "model.layers.51.mlp.experts.12.up_proj", "model.layers.51.mlp.experts.13.up_proj", "model.layers.51.mlp.experts.14.up_proj", "model.layers.51.mlp.experts.15.up_proj", "model.layers.51.mlp.experts.16.up_proj", "model.layers.51.mlp.experts.17.up_proj", "model.layers.51.mlp.experts.18.up_proj", "model.layers.51.mlp.experts.19.up_proj", "model.layers.51.mlp.experts.20.up_proj", "model.layers.51.mlp.experts.21.up_proj", "model.layers.51.mlp.experts.22.up_proj", "model.layers.51.mlp.experts.23.up_proj", "model.layers.51.mlp.experts.24.up_proj", "model.layers.51.mlp.experts.25.up_proj", "model.layers.51.mlp.experts.26.up_proj", "model.layers.51.mlp.experts.27.up_proj", "model.layers.51.mlp.experts.28.up_proj", "model.layers.51.mlp.experts.29.up_proj", "model.layers.51.mlp.experts.30.up_proj", "model.layers.51.mlp.experts.31.up_proj", "model.layers.51.mlp.experts.32.up_proj", "model.layers.51.mlp.experts.33.up_proj", "model.layers.51.mlp.experts.34.up_proj", "model.layers.51.mlp.experts.35.up_proj", "model.layers.51.mlp.experts.36.up_proj", "model.layers.51.mlp.experts.37.up_proj", "model.layers.51.mlp.experts.38.up_proj", "model.layers.51.mlp.experts.39.up_proj", "model.layers.51.mlp.experts.40.up_proj", "model.layers.51.mlp.experts.41.up_proj", "model.layers.51.mlp.experts.42.up_proj", "model.layers.51.mlp.experts.43.up_proj", "model.layers.51.mlp.experts.44.up_proj", "model.layers.51.mlp.experts.45.up_proj", "model.layers.51.mlp.experts.46.up_proj", "model.layers.51.mlp.experts.47.up_proj", "model.layers.51.mlp.experts.48.up_proj", "model.layers.51.mlp.experts.49.up_proj", "model.layers.51.mlp.experts.50.up_proj", "model.layers.51.mlp.experts.51.up_proj", "model.layers.51.mlp.experts.52.up_proj", "model.layers.51.mlp.experts.53.up_proj", "model.layers.51.mlp.experts.54.up_proj", "model.layers.51.mlp.experts.55.up_proj", "model.layers.51.mlp.experts.56.up_proj", "model.layers.51.mlp.experts.57.up_proj", "model.layers.51.mlp.experts.58.up_proj", "model.layers.51.mlp.experts.59.up_proj", "model.layers.51.mlp.experts.60.up_proj", "model.layers.51.mlp.experts.61.up_proj", "model.layers.51.mlp.experts.62.up_proj", "model.layers.51.mlp.experts.63.up_proj", "model.layers.51.mlp.experts.64.up_proj", "model.layers.51.mlp.experts.65.up_proj", "model.layers.51.mlp.experts.66.up_proj", "model.layers.51.mlp.experts.67.up_proj", "model.layers.51.mlp.experts.68.up_proj", "model.layers.51.mlp.experts.69.up_proj", "model.layers.51.mlp.experts.70.up_proj", "model.layers.51.mlp.experts.71.up_proj", "model.layers.51.mlp.experts.72.up_proj", "model.layers.51.mlp.experts.73.up_proj", "model.layers.51.mlp.experts.74.up_proj", "model.layers.51.mlp.experts.75.up_proj", "model.layers.51.mlp.experts.76.up_proj", "model.layers.51.mlp.experts.77.up_proj", "model.layers.51.mlp.experts.78.up_proj", "model.layers.51.mlp.experts.79.up_proj", "model.layers.51.mlp.experts.80.up_proj", "model.layers.51.mlp.experts.81.up_proj", "model.layers.51.mlp.experts.82.up_proj", "model.layers.51.mlp.experts.83.up_proj", "model.layers.51.mlp.experts.84.up_proj", "model.layers.51.mlp.experts.85.up_proj", "model.layers.51.mlp.experts.86.up_proj", "model.layers.51.mlp.experts.87.up_proj", "model.layers.51.mlp.experts.88.up_proj", "model.layers.51.mlp.experts.89.up_proj", "model.layers.51.mlp.experts.90.up_proj", "model.layers.51.mlp.experts.91.up_proj", "model.layers.51.mlp.experts.92.up_proj", "model.layers.51.mlp.experts.93.up_proj", "model.layers.51.mlp.experts.94.up_proj", "model.layers.51.mlp.experts.95.up_proj", "model.layers.51.mlp.experts.96.up_proj", "model.layers.51.mlp.experts.97.up_proj", "model.layers.51.mlp.experts.98.up_proj", "model.layers.51.mlp.experts.99.up_proj", "model.layers.51.mlp.experts.100.up_proj", "model.layers.51.mlp.experts.101.up_proj", "model.layers.51.mlp.experts.102.up_proj", "model.layers.51.mlp.experts.103.up_proj", "model.layers.51.mlp.experts.104.up_proj", "model.layers.51.mlp.experts.105.up_proj", "model.layers.51.mlp.experts.106.up_proj", "model.layers.51.mlp.experts.107.up_proj", "model.layers.51.mlp.experts.108.up_proj", "model.layers.51.mlp.experts.109.up_proj", "model.layers.51.mlp.experts.110.up_proj", "model.layers.51.mlp.experts.111.up_proj", "model.layers.51.mlp.experts.112.up_proj", "model.layers.51.mlp.experts.113.up_proj", "model.layers.51.mlp.experts.114.up_proj", "model.layers.51.mlp.experts.115.up_proj", "model.layers.51.mlp.experts.116.up_proj", "model.layers.51.mlp.experts.117.up_proj", "model.layers.51.mlp.experts.118.up_proj", "model.layers.51.mlp.experts.119.up_proj", "model.layers.51.mlp.experts.120.up_proj", "model.layers.51.mlp.experts.121.up_proj", "model.layers.51.mlp.experts.122.up_proj", "model.layers.51.mlp.experts.123.up_proj", "model.layers.51.mlp.experts.124.up_proj", "model.layers.51.mlp.experts.125.up_proj", "model.layers.51.mlp.experts.126.up_proj", "model.layers.51.mlp.experts.127.up_proj", "model.layers.51.mlp.experts.128.up_proj", "model.layers.51.mlp.experts.129.up_proj", "model.layers.51.mlp.experts.130.up_proj", "model.layers.51.mlp.experts.131.up_proj", "model.layers.51.mlp.experts.132.up_proj", "model.layers.51.mlp.experts.133.up_proj", "model.layers.51.mlp.experts.134.up_proj", "model.layers.51.mlp.experts.135.up_proj", "model.layers.51.mlp.experts.136.up_proj", "model.layers.51.mlp.experts.137.up_proj", "model.layers.51.mlp.experts.138.up_proj", "model.layers.51.mlp.experts.139.up_proj", "model.layers.51.mlp.experts.140.up_proj", "model.layers.51.mlp.experts.141.up_proj", "model.layers.51.mlp.experts.142.up_proj", "model.layers.51.mlp.experts.143.up_proj", "model.layers.51.mlp.experts.144.up_proj", "model.layers.51.mlp.experts.145.up_proj", "model.layers.51.mlp.experts.146.up_proj", "model.layers.51.mlp.experts.147.up_proj", "model.layers.51.mlp.experts.148.up_proj", "model.layers.51.mlp.experts.149.up_proj", "model.layers.51.mlp.experts.150.up_proj", "model.layers.51.mlp.experts.151.up_proj", "model.layers.51.mlp.experts.152.up_proj", "model.layers.51.mlp.experts.153.up_proj", "model.layers.51.mlp.experts.154.up_proj", "model.layers.51.mlp.experts.155.up_proj", "model.layers.51.mlp.experts.156.up_proj", "model.layers.51.mlp.experts.157.up_proj", "model.layers.51.mlp.experts.158.up_proj", "model.layers.51.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.1598225682973645e-06, "dbits": 5033164800 } ] }, { "idx": 308, "layers": [ "model.layers.51.mlp.experts.0.down_proj", "model.layers.51.mlp.experts.1.down_proj", "model.layers.51.mlp.experts.2.down_proj", "model.layers.51.mlp.experts.3.down_proj", "model.layers.51.mlp.experts.4.down_proj", "model.layers.51.mlp.experts.5.down_proj", "model.layers.51.mlp.experts.6.down_proj", "model.layers.51.mlp.experts.7.down_proj", "model.layers.51.mlp.experts.8.down_proj", "model.layers.51.mlp.experts.9.down_proj", "model.layers.51.mlp.experts.10.down_proj", "model.layers.51.mlp.experts.11.down_proj", "model.layers.51.mlp.experts.12.down_proj", "model.layers.51.mlp.experts.13.down_proj", "model.layers.51.mlp.experts.14.down_proj", "model.layers.51.mlp.experts.15.down_proj", "model.layers.51.mlp.experts.16.down_proj", "model.layers.51.mlp.experts.17.down_proj", "model.layers.51.mlp.experts.18.down_proj", "model.layers.51.mlp.experts.19.down_proj", "model.layers.51.mlp.experts.20.down_proj", "model.layers.51.mlp.experts.21.down_proj", "model.layers.51.mlp.experts.22.down_proj", "model.layers.51.mlp.experts.23.down_proj", "model.layers.51.mlp.experts.24.down_proj", "model.layers.51.mlp.experts.25.down_proj", "model.layers.51.mlp.experts.26.down_proj", "model.layers.51.mlp.experts.27.down_proj", "model.layers.51.mlp.experts.28.down_proj", "model.layers.51.mlp.experts.29.down_proj", "model.layers.51.mlp.experts.30.down_proj", "model.layers.51.mlp.experts.31.down_proj", "model.layers.51.mlp.experts.32.down_proj", "model.layers.51.mlp.experts.33.down_proj", "model.layers.51.mlp.experts.34.down_proj", "model.layers.51.mlp.experts.35.down_proj", "model.layers.51.mlp.experts.36.down_proj", "model.layers.51.mlp.experts.37.down_proj", "model.layers.51.mlp.experts.38.down_proj", "model.layers.51.mlp.experts.39.down_proj", "model.layers.51.mlp.experts.40.down_proj", "model.layers.51.mlp.experts.41.down_proj", "model.layers.51.mlp.experts.42.down_proj", "model.layers.51.mlp.experts.43.down_proj", "model.layers.51.mlp.experts.44.down_proj", "model.layers.51.mlp.experts.45.down_proj", "model.layers.51.mlp.experts.46.down_proj", "model.layers.51.mlp.experts.47.down_proj", "model.layers.51.mlp.experts.48.down_proj", "model.layers.51.mlp.experts.49.down_proj", "model.layers.51.mlp.experts.50.down_proj", "model.layers.51.mlp.experts.51.down_proj", "model.layers.51.mlp.experts.52.down_proj", "model.layers.51.mlp.experts.53.down_proj", "model.layers.51.mlp.experts.54.down_proj", "model.layers.51.mlp.experts.55.down_proj", "model.layers.51.mlp.experts.56.down_proj", "model.layers.51.mlp.experts.57.down_proj", "model.layers.51.mlp.experts.58.down_proj", "model.layers.51.mlp.experts.59.down_proj", "model.layers.51.mlp.experts.60.down_proj", "model.layers.51.mlp.experts.61.down_proj", "model.layers.51.mlp.experts.62.down_proj", "model.layers.51.mlp.experts.63.down_proj", "model.layers.51.mlp.experts.64.down_proj", "model.layers.51.mlp.experts.65.down_proj", "model.layers.51.mlp.experts.66.down_proj", "model.layers.51.mlp.experts.67.down_proj", "model.layers.51.mlp.experts.68.down_proj", "model.layers.51.mlp.experts.69.down_proj", "model.layers.51.mlp.experts.70.down_proj", "model.layers.51.mlp.experts.71.down_proj", "model.layers.51.mlp.experts.72.down_proj", "model.layers.51.mlp.experts.73.down_proj", "model.layers.51.mlp.experts.74.down_proj", "model.layers.51.mlp.experts.75.down_proj", "model.layers.51.mlp.experts.76.down_proj", "model.layers.51.mlp.experts.77.down_proj", "model.layers.51.mlp.experts.78.down_proj", "model.layers.51.mlp.experts.79.down_proj", "model.layers.51.mlp.experts.80.down_proj", "model.layers.51.mlp.experts.81.down_proj", "model.layers.51.mlp.experts.82.down_proj", "model.layers.51.mlp.experts.83.down_proj", "model.layers.51.mlp.experts.84.down_proj", "model.layers.51.mlp.experts.85.down_proj", "model.layers.51.mlp.experts.86.down_proj", "model.layers.51.mlp.experts.87.down_proj", "model.layers.51.mlp.experts.88.down_proj", "model.layers.51.mlp.experts.89.down_proj", "model.layers.51.mlp.experts.90.down_proj", "model.layers.51.mlp.experts.91.down_proj", "model.layers.51.mlp.experts.92.down_proj", "model.layers.51.mlp.experts.93.down_proj", "model.layers.51.mlp.experts.94.down_proj", "model.layers.51.mlp.experts.95.down_proj", "model.layers.51.mlp.experts.96.down_proj", "model.layers.51.mlp.experts.97.down_proj", "model.layers.51.mlp.experts.98.down_proj", "model.layers.51.mlp.experts.99.down_proj", "model.layers.51.mlp.experts.100.down_proj", "model.layers.51.mlp.experts.101.down_proj", "model.layers.51.mlp.experts.102.down_proj", "model.layers.51.mlp.experts.103.down_proj", "model.layers.51.mlp.experts.104.down_proj", "model.layers.51.mlp.experts.105.down_proj", "model.layers.51.mlp.experts.106.down_proj", "model.layers.51.mlp.experts.107.down_proj", "model.layers.51.mlp.experts.108.down_proj", "model.layers.51.mlp.experts.109.down_proj", "model.layers.51.mlp.experts.110.down_proj", "model.layers.51.mlp.experts.111.down_proj", "model.layers.51.mlp.experts.112.down_proj", "model.layers.51.mlp.experts.113.down_proj", "model.layers.51.mlp.experts.114.down_proj", "model.layers.51.mlp.experts.115.down_proj", "model.layers.51.mlp.experts.116.down_proj", "model.layers.51.mlp.experts.117.down_proj", "model.layers.51.mlp.experts.118.down_proj", "model.layers.51.mlp.experts.119.down_proj", "model.layers.51.mlp.experts.120.down_proj", "model.layers.51.mlp.experts.121.down_proj", "model.layers.51.mlp.experts.122.down_proj", "model.layers.51.mlp.experts.123.down_proj", "model.layers.51.mlp.experts.124.down_proj", "model.layers.51.mlp.experts.125.down_proj", "model.layers.51.mlp.experts.126.down_proj", "model.layers.51.mlp.experts.127.down_proj", "model.layers.51.mlp.experts.128.down_proj", "model.layers.51.mlp.experts.129.down_proj", "model.layers.51.mlp.experts.130.down_proj", "model.layers.51.mlp.experts.131.down_proj", "model.layers.51.mlp.experts.132.down_proj", "model.layers.51.mlp.experts.133.down_proj", "model.layers.51.mlp.experts.134.down_proj", "model.layers.51.mlp.experts.135.down_proj", "model.layers.51.mlp.experts.136.down_proj", "model.layers.51.mlp.experts.137.down_proj", "model.layers.51.mlp.experts.138.down_proj", "model.layers.51.mlp.experts.139.down_proj", "model.layers.51.mlp.experts.140.down_proj", "model.layers.51.mlp.experts.141.down_proj", "model.layers.51.mlp.experts.142.down_proj", "model.layers.51.mlp.experts.143.down_proj", "model.layers.51.mlp.experts.144.down_proj", "model.layers.51.mlp.experts.145.down_proj", "model.layers.51.mlp.experts.146.down_proj", "model.layers.51.mlp.experts.147.down_proj", "model.layers.51.mlp.experts.148.down_proj", "model.layers.51.mlp.experts.149.down_proj", "model.layers.51.mlp.experts.150.down_proj", "model.layers.51.mlp.experts.151.down_proj", "model.layers.51.mlp.experts.152.down_proj", "model.layers.51.mlp.experts.153.down_proj", "model.layers.51.mlp.experts.154.down_proj", "model.layers.51.mlp.experts.155.down_proj", "model.layers.51.mlp.experts.156.down_proj", "model.layers.51.mlp.experts.157.down_proj", "model.layers.51.mlp.experts.158.down_proj", "model.layers.51.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.897718852385922e-06, "dbits": 2516582400 } ] }, { "idx": 309, "layers": [ "model.layers.52.self_attn.q_proj" ], "candidates": [ { "dkld": 4.243772127665536e-06, "dbits": 125829120 } ] }, { "idx": 310, "layers": [ "model.layers.52.self_attn.k_proj", "model.layers.52.self_attn.v_proj" ], "candidates": [ { "dkld": 5.763457738794343e-06, "dbits": 20971520 } ] }, { "idx": 311, "layers": [ "model.layers.52.self_attn.o_proj" ], "candidates": [ { "dkld": 2.20424262806771e-06, "dbits": 125829120 } ] }, { "idx": 312, "layers": [ "model.layers.52.mlp.shared_experts.gate_proj", "model.layers.52.mlp.shared_experts.up_proj", "model.layers.52.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.6783905923366763e-06, "dbits": 47185920 } ] }, { "idx": 313, "layers": [ "model.layers.52.mlp.experts.0.gate_proj", "model.layers.52.mlp.experts.1.gate_proj", "model.layers.52.mlp.experts.2.gate_proj", "model.layers.52.mlp.experts.3.gate_proj", "model.layers.52.mlp.experts.4.gate_proj", "model.layers.52.mlp.experts.5.gate_proj", "model.layers.52.mlp.experts.6.gate_proj", "model.layers.52.mlp.experts.7.gate_proj", "model.layers.52.mlp.experts.8.gate_proj", "model.layers.52.mlp.experts.9.gate_proj", "model.layers.52.mlp.experts.10.gate_proj", "model.layers.52.mlp.experts.11.gate_proj", "model.layers.52.mlp.experts.12.gate_proj", "model.layers.52.mlp.experts.13.gate_proj", "model.layers.52.mlp.experts.14.gate_proj", "model.layers.52.mlp.experts.15.gate_proj", "model.layers.52.mlp.experts.16.gate_proj", "model.layers.52.mlp.experts.17.gate_proj", "model.layers.52.mlp.experts.18.gate_proj", "model.layers.52.mlp.experts.19.gate_proj", "model.layers.52.mlp.experts.20.gate_proj", "model.layers.52.mlp.experts.21.gate_proj", "model.layers.52.mlp.experts.22.gate_proj", "model.layers.52.mlp.experts.23.gate_proj", "model.layers.52.mlp.experts.24.gate_proj", "model.layers.52.mlp.experts.25.gate_proj", "model.layers.52.mlp.experts.26.gate_proj", "model.layers.52.mlp.experts.27.gate_proj", "model.layers.52.mlp.experts.28.gate_proj", "model.layers.52.mlp.experts.29.gate_proj", "model.layers.52.mlp.experts.30.gate_proj", "model.layers.52.mlp.experts.31.gate_proj", "model.layers.52.mlp.experts.32.gate_proj", "model.layers.52.mlp.experts.33.gate_proj", "model.layers.52.mlp.experts.34.gate_proj", "model.layers.52.mlp.experts.35.gate_proj", "model.layers.52.mlp.experts.36.gate_proj", "model.layers.52.mlp.experts.37.gate_proj", "model.layers.52.mlp.experts.38.gate_proj", "model.layers.52.mlp.experts.39.gate_proj", "model.layers.52.mlp.experts.40.gate_proj", "model.layers.52.mlp.experts.41.gate_proj", "model.layers.52.mlp.experts.42.gate_proj", "model.layers.52.mlp.experts.43.gate_proj", "model.layers.52.mlp.experts.44.gate_proj", "model.layers.52.mlp.experts.45.gate_proj", "model.layers.52.mlp.experts.46.gate_proj", "model.layers.52.mlp.experts.47.gate_proj", "model.layers.52.mlp.experts.48.gate_proj", "model.layers.52.mlp.experts.49.gate_proj", "model.layers.52.mlp.experts.50.gate_proj", "model.layers.52.mlp.experts.51.gate_proj", "model.layers.52.mlp.experts.52.gate_proj", "model.layers.52.mlp.experts.53.gate_proj", "model.layers.52.mlp.experts.54.gate_proj", "model.layers.52.mlp.experts.55.gate_proj", "model.layers.52.mlp.experts.56.gate_proj", "model.layers.52.mlp.experts.57.gate_proj", "model.layers.52.mlp.experts.58.gate_proj", "model.layers.52.mlp.experts.59.gate_proj", "model.layers.52.mlp.experts.60.gate_proj", "model.layers.52.mlp.experts.61.gate_proj", "model.layers.52.mlp.experts.62.gate_proj", "model.layers.52.mlp.experts.63.gate_proj", "model.layers.52.mlp.experts.64.gate_proj", "model.layers.52.mlp.experts.65.gate_proj", "model.layers.52.mlp.experts.66.gate_proj", "model.layers.52.mlp.experts.67.gate_proj", "model.layers.52.mlp.experts.68.gate_proj", "model.layers.52.mlp.experts.69.gate_proj", "model.layers.52.mlp.experts.70.gate_proj", "model.layers.52.mlp.experts.71.gate_proj", "model.layers.52.mlp.experts.72.gate_proj", "model.layers.52.mlp.experts.73.gate_proj", "model.layers.52.mlp.experts.74.gate_proj", "model.layers.52.mlp.experts.75.gate_proj", "model.layers.52.mlp.experts.76.gate_proj", "model.layers.52.mlp.experts.77.gate_proj", "model.layers.52.mlp.experts.78.gate_proj", "model.layers.52.mlp.experts.79.gate_proj", "model.layers.52.mlp.experts.80.gate_proj", "model.layers.52.mlp.experts.81.gate_proj", "model.layers.52.mlp.experts.82.gate_proj", "model.layers.52.mlp.experts.83.gate_proj", "model.layers.52.mlp.experts.84.gate_proj", "model.layers.52.mlp.experts.85.gate_proj", "model.layers.52.mlp.experts.86.gate_proj", "model.layers.52.mlp.experts.87.gate_proj", "model.layers.52.mlp.experts.88.gate_proj", "model.layers.52.mlp.experts.89.gate_proj", "model.layers.52.mlp.experts.90.gate_proj", "model.layers.52.mlp.experts.91.gate_proj", "model.layers.52.mlp.experts.92.gate_proj", "model.layers.52.mlp.experts.93.gate_proj", "model.layers.52.mlp.experts.94.gate_proj", "model.layers.52.mlp.experts.95.gate_proj", "model.layers.52.mlp.experts.96.gate_proj", "model.layers.52.mlp.experts.97.gate_proj", "model.layers.52.mlp.experts.98.gate_proj", "model.layers.52.mlp.experts.99.gate_proj", "model.layers.52.mlp.experts.100.gate_proj", "model.layers.52.mlp.experts.101.gate_proj", "model.layers.52.mlp.experts.102.gate_proj", "model.layers.52.mlp.experts.103.gate_proj", "model.layers.52.mlp.experts.104.gate_proj", "model.layers.52.mlp.experts.105.gate_proj", "model.layers.52.mlp.experts.106.gate_proj", "model.layers.52.mlp.experts.107.gate_proj", "model.layers.52.mlp.experts.108.gate_proj", "model.layers.52.mlp.experts.109.gate_proj", "model.layers.52.mlp.experts.110.gate_proj", "model.layers.52.mlp.experts.111.gate_proj", "model.layers.52.mlp.experts.112.gate_proj", "model.layers.52.mlp.experts.113.gate_proj", "model.layers.52.mlp.experts.114.gate_proj", "model.layers.52.mlp.experts.115.gate_proj", "model.layers.52.mlp.experts.116.gate_proj", "model.layers.52.mlp.experts.117.gate_proj", "model.layers.52.mlp.experts.118.gate_proj", "model.layers.52.mlp.experts.119.gate_proj", "model.layers.52.mlp.experts.120.gate_proj", "model.layers.52.mlp.experts.121.gate_proj", "model.layers.52.mlp.experts.122.gate_proj", "model.layers.52.mlp.experts.123.gate_proj", "model.layers.52.mlp.experts.124.gate_proj", "model.layers.52.mlp.experts.125.gate_proj", "model.layers.52.mlp.experts.126.gate_proj", "model.layers.52.mlp.experts.127.gate_proj", "model.layers.52.mlp.experts.128.gate_proj", "model.layers.52.mlp.experts.129.gate_proj", "model.layers.52.mlp.experts.130.gate_proj", "model.layers.52.mlp.experts.131.gate_proj", "model.layers.52.mlp.experts.132.gate_proj", "model.layers.52.mlp.experts.133.gate_proj", "model.layers.52.mlp.experts.134.gate_proj", "model.layers.52.mlp.experts.135.gate_proj", "model.layers.52.mlp.experts.136.gate_proj", "model.layers.52.mlp.experts.137.gate_proj", "model.layers.52.mlp.experts.138.gate_proj", "model.layers.52.mlp.experts.139.gate_proj", "model.layers.52.mlp.experts.140.gate_proj", "model.layers.52.mlp.experts.141.gate_proj", "model.layers.52.mlp.experts.142.gate_proj", "model.layers.52.mlp.experts.143.gate_proj", "model.layers.52.mlp.experts.144.gate_proj", "model.layers.52.mlp.experts.145.gate_proj", "model.layers.52.mlp.experts.146.gate_proj", "model.layers.52.mlp.experts.147.gate_proj", "model.layers.52.mlp.experts.148.gate_proj", "model.layers.52.mlp.experts.149.gate_proj", "model.layers.52.mlp.experts.150.gate_proj", "model.layers.52.mlp.experts.151.gate_proj", "model.layers.52.mlp.experts.152.gate_proj", "model.layers.52.mlp.experts.153.gate_proj", "model.layers.52.mlp.experts.154.gate_proj", "model.layers.52.mlp.experts.155.gate_proj", "model.layers.52.mlp.experts.156.gate_proj", "model.layers.52.mlp.experts.157.gate_proj", "model.layers.52.mlp.experts.158.gate_proj", "model.layers.52.mlp.experts.159.gate_proj", "model.layers.52.mlp.experts.0.up_proj", "model.layers.52.mlp.experts.1.up_proj", "model.layers.52.mlp.experts.2.up_proj", "model.layers.52.mlp.experts.3.up_proj", "model.layers.52.mlp.experts.4.up_proj", "model.layers.52.mlp.experts.5.up_proj", "model.layers.52.mlp.experts.6.up_proj", "model.layers.52.mlp.experts.7.up_proj", "model.layers.52.mlp.experts.8.up_proj", "model.layers.52.mlp.experts.9.up_proj", "model.layers.52.mlp.experts.10.up_proj", "model.layers.52.mlp.experts.11.up_proj", "model.layers.52.mlp.experts.12.up_proj", "model.layers.52.mlp.experts.13.up_proj", "model.layers.52.mlp.experts.14.up_proj", "model.layers.52.mlp.experts.15.up_proj", "model.layers.52.mlp.experts.16.up_proj", "model.layers.52.mlp.experts.17.up_proj", "model.layers.52.mlp.experts.18.up_proj", "model.layers.52.mlp.experts.19.up_proj", "model.layers.52.mlp.experts.20.up_proj", "model.layers.52.mlp.experts.21.up_proj", "model.layers.52.mlp.experts.22.up_proj", "model.layers.52.mlp.experts.23.up_proj", "model.layers.52.mlp.experts.24.up_proj", "model.layers.52.mlp.experts.25.up_proj", "model.layers.52.mlp.experts.26.up_proj", "model.layers.52.mlp.experts.27.up_proj", "model.layers.52.mlp.experts.28.up_proj", "model.layers.52.mlp.experts.29.up_proj", "model.layers.52.mlp.experts.30.up_proj", "model.layers.52.mlp.experts.31.up_proj", "model.layers.52.mlp.experts.32.up_proj", "model.layers.52.mlp.experts.33.up_proj", "model.layers.52.mlp.experts.34.up_proj", "model.layers.52.mlp.experts.35.up_proj", "model.layers.52.mlp.experts.36.up_proj", "model.layers.52.mlp.experts.37.up_proj", "model.layers.52.mlp.experts.38.up_proj", "model.layers.52.mlp.experts.39.up_proj", "model.layers.52.mlp.experts.40.up_proj", "model.layers.52.mlp.experts.41.up_proj", "model.layers.52.mlp.experts.42.up_proj", "model.layers.52.mlp.experts.43.up_proj", "model.layers.52.mlp.experts.44.up_proj", "model.layers.52.mlp.experts.45.up_proj", "model.layers.52.mlp.experts.46.up_proj", "model.layers.52.mlp.experts.47.up_proj", "model.layers.52.mlp.experts.48.up_proj", "model.layers.52.mlp.experts.49.up_proj", "model.layers.52.mlp.experts.50.up_proj", "model.layers.52.mlp.experts.51.up_proj", "model.layers.52.mlp.experts.52.up_proj", "model.layers.52.mlp.experts.53.up_proj", "model.layers.52.mlp.experts.54.up_proj", "model.layers.52.mlp.experts.55.up_proj", "model.layers.52.mlp.experts.56.up_proj", "model.layers.52.mlp.experts.57.up_proj", "model.layers.52.mlp.experts.58.up_proj", "model.layers.52.mlp.experts.59.up_proj", "model.layers.52.mlp.experts.60.up_proj", "model.layers.52.mlp.experts.61.up_proj", "model.layers.52.mlp.experts.62.up_proj", "model.layers.52.mlp.experts.63.up_proj", "model.layers.52.mlp.experts.64.up_proj", "model.layers.52.mlp.experts.65.up_proj", "model.layers.52.mlp.experts.66.up_proj", "model.layers.52.mlp.experts.67.up_proj", "model.layers.52.mlp.experts.68.up_proj", "model.layers.52.mlp.experts.69.up_proj", "model.layers.52.mlp.experts.70.up_proj", "model.layers.52.mlp.experts.71.up_proj", "model.layers.52.mlp.experts.72.up_proj", "model.layers.52.mlp.experts.73.up_proj", "model.layers.52.mlp.experts.74.up_proj", "model.layers.52.mlp.experts.75.up_proj", "model.layers.52.mlp.experts.76.up_proj", "model.layers.52.mlp.experts.77.up_proj", "model.layers.52.mlp.experts.78.up_proj", "model.layers.52.mlp.experts.79.up_proj", "model.layers.52.mlp.experts.80.up_proj", "model.layers.52.mlp.experts.81.up_proj", "model.layers.52.mlp.experts.82.up_proj", "model.layers.52.mlp.experts.83.up_proj", "model.layers.52.mlp.experts.84.up_proj", "model.layers.52.mlp.experts.85.up_proj", "model.layers.52.mlp.experts.86.up_proj", "model.layers.52.mlp.experts.87.up_proj", "model.layers.52.mlp.experts.88.up_proj", "model.layers.52.mlp.experts.89.up_proj", "model.layers.52.mlp.experts.90.up_proj", "model.layers.52.mlp.experts.91.up_proj", "model.layers.52.mlp.experts.92.up_proj", "model.layers.52.mlp.experts.93.up_proj", "model.layers.52.mlp.experts.94.up_proj", "model.layers.52.mlp.experts.95.up_proj", "model.layers.52.mlp.experts.96.up_proj", "model.layers.52.mlp.experts.97.up_proj", "model.layers.52.mlp.experts.98.up_proj", "model.layers.52.mlp.experts.99.up_proj", "model.layers.52.mlp.experts.100.up_proj", "model.layers.52.mlp.experts.101.up_proj", "model.layers.52.mlp.experts.102.up_proj", "model.layers.52.mlp.experts.103.up_proj", "model.layers.52.mlp.experts.104.up_proj", "model.layers.52.mlp.experts.105.up_proj", "model.layers.52.mlp.experts.106.up_proj", "model.layers.52.mlp.experts.107.up_proj", "model.layers.52.mlp.experts.108.up_proj", "model.layers.52.mlp.experts.109.up_proj", "model.layers.52.mlp.experts.110.up_proj", "model.layers.52.mlp.experts.111.up_proj", "model.layers.52.mlp.experts.112.up_proj", "model.layers.52.mlp.experts.113.up_proj", "model.layers.52.mlp.experts.114.up_proj", "model.layers.52.mlp.experts.115.up_proj", "model.layers.52.mlp.experts.116.up_proj", "model.layers.52.mlp.experts.117.up_proj", "model.layers.52.mlp.experts.118.up_proj", "model.layers.52.mlp.experts.119.up_proj", "model.layers.52.mlp.experts.120.up_proj", "model.layers.52.mlp.experts.121.up_proj", "model.layers.52.mlp.experts.122.up_proj", "model.layers.52.mlp.experts.123.up_proj", "model.layers.52.mlp.experts.124.up_proj", "model.layers.52.mlp.experts.125.up_proj", "model.layers.52.mlp.experts.126.up_proj", "model.layers.52.mlp.experts.127.up_proj", "model.layers.52.mlp.experts.128.up_proj", "model.layers.52.mlp.experts.129.up_proj", "model.layers.52.mlp.experts.130.up_proj", "model.layers.52.mlp.experts.131.up_proj", "model.layers.52.mlp.experts.132.up_proj", "model.layers.52.mlp.experts.133.up_proj", "model.layers.52.mlp.experts.134.up_proj", "model.layers.52.mlp.experts.135.up_proj", "model.layers.52.mlp.experts.136.up_proj", "model.layers.52.mlp.experts.137.up_proj", "model.layers.52.mlp.experts.138.up_proj", "model.layers.52.mlp.experts.139.up_proj", "model.layers.52.mlp.experts.140.up_proj", "model.layers.52.mlp.experts.141.up_proj", "model.layers.52.mlp.experts.142.up_proj", "model.layers.52.mlp.experts.143.up_proj", "model.layers.52.mlp.experts.144.up_proj", "model.layers.52.mlp.experts.145.up_proj", "model.layers.52.mlp.experts.146.up_proj", "model.layers.52.mlp.experts.147.up_proj", "model.layers.52.mlp.experts.148.up_proj", "model.layers.52.mlp.experts.149.up_proj", "model.layers.52.mlp.experts.150.up_proj", "model.layers.52.mlp.experts.151.up_proj", "model.layers.52.mlp.experts.152.up_proj", "model.layers.52.mlp.experts.153.up_proj", "model.layers.52.mlp.experts.154.up_proj", "model.layers.52.mlp.experts.155.up_proj", "model.layers.52.mlp.experts.156.up_proj", "model.layers.52.mlp.experts.157.up_proj", "model.layers.52.mlp.experts.158.up_proj", "model.layers.52.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.801116741262333e-06, "dbits": 5033164800 } ] }, { "idx": 314, "layers": [ "model.layers.52.mlp.experts.0.down_proj", "model.layers.52.mlp.experts.1.down_proj", "model.layers.52.mlp.experts.2.down_proj", "model.layers.52.mlp.experts.3.down_proj", "model.layers.52.mlp.experts.4.down_proj", "model.layers.52.mlp.experts.5.down_proj", "model.layers.52.mlp.experts.6.down_proj", "model.layers.52.mlp.experts.7.down_proj", "model.layers.52.mlp.experts.8.down_proj", "model.layers.52.mlp.experts.9.down_proj", "model.layers.52.mlp.experts.10.down_proj", "model.layers.52.mlp.experts.11.down_proj", "model.layers.52.mlp.experts.12.down_proj", "model.layers.52.mlp.experts.13.down_proj", "model.layers.52.mlp.experts.14.down_proj", "model.layers.52.mlp.experts.15.down_proj", "model.layers.52.mlp.experts.16.down_proj", "model.layers.52.mlp.experts.17.down_proj", "model.layers.52.mlp.experts.18.down_proj", "model.layers.52.mlp.experts.19.down_proj", "model.layers.52.mlp.experts.20.down_proj", "model.layers.52.mlp.experts.21.down_proj", "model.layers.52.mlp.experts.22.down_proj", "model.layers.52.mlp.experts.23.down_proj", "model.layers.52.mlp.experts.24.down_proj", "model.layers.52.mlp.experts.25.down_proj", "model.layers.52.mlp.experts.26.down_proj", "model.layers.52.mlp.experts.27.down_proj", "model.layers.52.mlp.experts.28.down_proj", "model.layers.52.mlp.experts.29.down_proj", "model.layers.52.mlp.experts.30.down_proj", "model.layers.52.mlp.experts.31.down_proj", "model.layers.52.mlp.experts.32.down_proj", "model.layers.52.mlp.experts.33.down_proj", "model.layers.52.mlp.experts.34.down_proj", "model.layers.52.mlp.experts.35.down_proj", "model.layers.52.mlp.experts.36.down_proj", "model.layers.52.mlp.experts.37.down_proj", "model.layers.52.mlp.experts.38.down_proj", "model.layers.52.mlp.experts.39.down_proj", "model.layers.52.mlp.experts.40.down_proj", "model.layers.52.mlp.experts.41.down_proj", "model.layers.52.mlp.experts.42.down_proj", "model.layers.52.mlp.experts.43.down_proj", "model.layers.52.mlp.experts.44.down_proj", "model.layers.52.mlp.experts.45.down_proj", "model.layers.52.mlp.experts.46.down_proj", "model.layers.52.mlp.experts.47.down_proj", "model.layers.52.mlp.experts.48.down_proj", "model.layers.52.mlp.experts.49.down_proj", "model.layers.52.mlp.experts.50.down_proj", "model.layers.52.mlp.experts.51.down_proj", "model.layers.52.mlp.experts.52.down_proj", "model.layers.52.mlp.experts.53.down_proj", "model.layers.52.mlp.experts.54.down_proj", "model.layers.52.mlp.experts.55.down_proj", "model.layers.52.mlp.experts.56.down_proj", "model.layers.52.mlp.experts.57.down_proj", "model.layers.52.mlp.experts.58.down_proj", "model.layers.52.mlp.experts.59.down_proj", "model.layers.52.mlp.experts.60.down_proj", "model.layers.52.mlp.experts.61.down_proj", "model.layers.52.mlp.experts.62.down_proj", "model.layers.52.mlp.experts.63.down_proj", "model.layers.52.mlp.experts.64.down_proj", "model.layers.52.mlp.experts.65.down_proj", "model.layers.52.mlp.experts.66.down_proj", "model.layers.52.mlp.experts.67.down_proj", "model.layers.52.mlp.experts.68.down_proj", "model.layers.52.mlp.experts.69.down_proj", "model.layers.52.mlp.experts.70.down_proj", "model.layers.52.mlp.experts.71.down_proj", "model.layers.52.mlp.experts.72.down_proj", "model.layers.52.mlp.experts.73.down_proj", "model.layers.52.mlp.experts.74.down_proj", "model.layers.52.mlp.experts.75.down_proj", "model.layers.52.mlp.experts.76.down_proj", "model.layers.52.mlp.experts.77.down_proj", "model.layers.52.mlp.experts.78.down_proj", "model.layers.52.mlp.experts.79.down_proj", "model.layers.52.mlp.experts.80.down_proj", "model.layers.52.mlp.experts.81.down_proj", "model.layers.52.mlp.experts.82.down_proj", "model.layers.52.mlp.experts.83.down_proj", "model.layers.52.mlp.experts.84.down_proj", "model.layers.52.mlp.experts.85.down_proj", "model.layers.52.mlp.experts.86.down_proj", "model.layers.52.mlp.experts.87.down_proj", "model.layers.52.mlp.experts.88.down_proj", "model.layers.52.mlp.experts.89.down_proj", "model.layers.52.mlp.experts.90.down_proj", "model.layers.52.mlp.experts.91.down_proj", "model.layers.52.mlp.experts.92.down_proj", "model.layers.52.mlp.experts.93.down_proj", "model.layers.52.mlp.experts.94.down_proj", "model.layers.52.mlp.experts.95.down_proj", "model.layers.52.mlp.experts.96.down_proj", "model.layers.52.mlp.experts.97.down_proj", "model.layers.52.mlp.experts.98.down_proj", "model.layers.52.mlp.experts.99.down_proj", "model.layers.52.mlp.experts.100.down_proj", "model.layers.52.mlp.experts.101.down_proj", "model.layers.52.mlp.experts.102.down_proj", "model.layers.52.mlp.experts.103.down_proj", "model.layers.52.mlp.experts.104.down_proj", "model.layers.52.mlp.experts.105.down_proj", "model.layers.52.mlp.experts.106.down_proj", "model.layers.52.mlp.experts.107.down_proj", "model.layers.52.mlp.experts.108.down_proj", "model.layers.52.mlp.experts.109.down_proj", "model.layers.52.mlp.experts.110.down_proj", "model.layers.52.mlp.experts.111.down_proj", "model.layers.52.mlp.experts.112.down_proj", "model.layers.52.mlp.experts.113.down_proj", "model.layers.52.mlp.experts.114.down_proj", "model.layers.52.mlp.experts.115.down_proj", "model.layers.52.mlp.experts.116.down_proj", "model.layers.52.mlp.experts.117.down_proj", "model.layers.52.mlp.experts.118.down_proj", "model.layers.52.mlp.experts.119.down_proj", "model.layers.52.mlp.experts.120.down_proj", "model.layers.52.mlp.experts.121.down_proj", "model.layers.52.mlp.experts.122.down_proj", "model.layers.52.mlp.experts.123.down_proj", "model.layers.52.mlp.experts.124.down_proj", "model.layers.52.mlp.experts.125.down_proj", "model.layers.52.mlp.experts.126.down_proj", "model.layers.52.mlp.experts.127.down_proj", "model.layers.52.mlp.experts.128.down_proj", "model.layers.52.mlp.experts.129.down_proj", "model.layers.52.mlp.experts.130.down_proj", "model.layers.52.mlp.experts.131.down_proj", "model.layers.52.mlp.experts.132.down_proj", "model.layers.52.mlp.experts.133.down_proj", "model.layers.52.mlp.experts.134.down_proj", "model.layers.52.mlp.experts.135.down_proj", "model.layers.52.mlp.experts.136.down_proj", "model.layers.52.mlp.experts.137.down_proj", "model.layers.52.mlp.experts.138.down_proj", "model.layers.52.mlp.experts.139.down_proj", "model.layers.52.mlp.experts.140.down_proj", "model.layers.52.mlp.experts.141.down_proj", "model.layers.52.mlp.experts.142.down_proj", "model.layers.52.mlp.experts.143.down_proj", "model.layers.52.mlp.experts.144.down_proj", "model.layers.52.mlp.experts.145.down_proj", "model.layers.52.mlp.experts.146.down_proj", "model.layers.52.mlp.experts.147.down_proj", "model.layers.52.mlp.experts.148.down_proj", "model.layers.52.mlp.experts.149.down_proj", "model.layers.52.mlp.experts.150.down_proj", "model.layers.52.mlp.experts.151.down_proj", "model.layers.52.mlp.experts.152.down_proj", "model.layers.52.mlp.experts.153.down_proj", "model.layers.52.mlp.experts.154.down_proj", "model.layers.52.mlp.experts.155.down_proj", "model.layers.52.mlp.experts.156.down_proj", "model.layers.52.mlp.experts.157.down_proj", "model.layers.52.mlp.experts.158.down_proj", "model.layers.52.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 6.684963591396852e-06, "dbits": 2516582400 } ] }, { "idx": 315, "layers": [ "model.layers.53.self_attn.q_proj" ], "candidates": [ { "dkld": 2.2465887013822577e-06, "dbits": 125829120 } ] }, { "idx": 316, "layers": [ "model.layers.53.self_attn.k_proj", "model.layers.53.self_attn.v_proj" ], "candidates": [ { "dkld": 1.4760153135284781e-06, "dbits": 20971520 } ] }, { "idx": 317, "layers": [ "model.layers.53.self_attn.o_proj" ], "candidates": [ { "dkld": -2.4899694835766933e-06, "dbits": 125829120 } ] }, { "idx": 318, "layers": [ "model.layers.53.mlp.shared_experts.gate_proj", "model.layers.53.mlp.shared_experts.up_proj", "model.layers.53.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.6769197322430395e-07, "dbits": 47185920 } ] }, { "idx": 319, "layers": [ "model.layers.53.mlp.experts.0.gate_proj", "model.layers.53.mlp.experts.1.gate_proj", "model.layers.53.mlp.experts.2.gate_proj", "model.layers.53.mlp.experts.3.gate_proj", "model.layers.53.mlp.experts.4.gate_proj", "model.layers.53.mlp.experts.5.gate_proj", "model.layers.53.mlp.experts.6.gate_proj", "model.layers.53.mlp.experts.7.gate_proj", "model.layers.53.mlp.experts.8.gate_proj", "model.layers.53.mlp.experts.9.gate_proj", "model.layers.53.mlp.experts.10.gate_proj", "model.layers.53.mlp.experts.11.gate_proj", "model.layers.53.mlp.experts.12.gate_proj", "model.layers.53.mlp.experts.13.gate_proj", "model.layers.53.mlp.experts.14.gate_proj", "model.layers.53.mlp.experts.15.gate_proj", "model.layers.53.mlp.experts.16.gate_proj", "model.layers.53.mlp.experts.17.gate_proj", "model.layers.53.mlp.experts.18.gate_proj", "model.layers.53.mlp.experts.19.gate_proj", "model.layers.53.mlp.experts.20.gate_proj", "model.layers.53.mlp.experts.21.gate_proj", "model.layers.53.mlp.experts.22.gate_proj", "model.layers.53.mlp.experts.23.gate_proj", "model.layers.53.mlp.experts.24.gate_proj", "model.layers.53.mlp.experts.25.gate_proj", "model.layers.53.mlp.experts.26.gate_proj", "model.layers.53.mlp.experts.27.gate_proj", "model.layers.53.mlp.experts.28.gate_proj", "model.layers.53.mlp.experts.29.gate_proj", "model.layers.53.mlp.experts.30.gate_proj", "model.layers.53.mlp.experts.31.gate_proj", "model.layers.53.mlp.experts.32.gate_proj", "model.layers.53.mlp.experts.33.gate_proj", "model.layers.53.mlp.experts.34.gate_proj", "model.layers.53.mlp.experts.35.gate_proj", "model.layers.53.mlp.experts.36.gate_proj", "model.layers.53.mlp.experts.37.gate_proj", "model.layers.53.mlp.experts.38.gate_proj", "model.layers.53.mlp.experts.39.gate_proj", "model.layers.53.mlp.experts.40.gate_proj", "model.layers.53.mlp.experts.41.gate_proj", "model.layers.53.mlp.experts.42.gate_proj", "model.layers.53.mlp.experts.43.gate_proj", "model.layers.53.mlp.experts.44.gate_proj", "model.layers.53.mlp.experts.45.gate_proj", "model.layers.53.mlp.experts.46.gate_proj", "model.layers.53.mlp.experts.47.gate_proj", "model.layers.53.mlp.experts.48.gate_proj", "model.layers.53.mlp.experts.49.gate_proj", "model.layers.53.mlp.experts.50.gate_proj", "model.layers.53.mlp.experts.51.gate_proj", "model.layers.53.mlp.experts.52.gate_proj", "model.layers.53.mlp.experts.53.gate_proj", "model.layers.53.mlp.experts.54.gate_proj", "model.layers.53.mlp.experts.55.gate_proj", "model.layers.53.mlp.experts.56.gate_proj", "model.layers.53.mlp.experts.57.gate_proj", "model.layers.53.mlp.experts.58.gate_proj", "model.layers.53.mlp.experts.59.gate_proj", "model.layers.53.mlp.experts.60.gate_proj", "model.layers.53.mlp.experts.61.gate_proj", "model.layers.53.mlp.experts.62.gate_proj", "model.layers.53.mlp.experts.63.gate_proj", "model.layers.53.mlp.experts.64.gate_proj", "model.layers.53.mlp.experts.65.gate_proj", "model.layers.53.mlp.experts.66.gate_proj", "model.layers.53.mlp.experts.67.gate_proj", "model.layers.53.mlp.experts.68.gate_proj", "model.layers.53.mlp.experts.69.gate_proj", "model.layers.53.mlp.experts.70.gate_proj", "model.layers.53.mlp.experts.71.gate_proj", "model.layers.53.mlp.experts.72.gate_proj", "model.layers.53.mlp.experts.73.gate_proj", "model.layers.53.mlp.experts.74.gate_proj", "model.layers.53.mlp.experts.75.gate_proj", "model.layers.53.mlp.experts.76.gate_proj", "model.layers.53.mlp.experts.77.gate_proj", "model.layers.53.mlp.experts.78.gate_proj", "model.layers.53.mlp.experts.79.gate_proj", "model.layers.53.mlp.experts.80.gate_proj", "model.layers.53.mlp.experts.81.gate_proj", "model.layers.53.mlp.experts.82.gate_proj", "model.layers.53.mlp.experts.83.gate_proj", "model.layers.53.mlp.experts.84.gate_proj", "model.layers.53.mlp.experts.85.gate_proj", "model.layers.53.mlp.experts.86.gate_proj", "model.layers.53.mlp.experts.87.gate_proj", "model.layers.53.mlp.experts.88.gate_proj", "model.layers.53.mlp.experts.89.gate_proj", "model.layers.53.mlp.experts.90.gate_proj", "model.layers.53.mlp.experts.91.gate_proj", "model.layers.53.mlp.experts.92.gate_proj", "model.layers.53.mlp.experts.93.gate_proj", "model.layers.53.mlp.experts.94.gate_proj", "model.layers.53.mlp.experts.95.gate_proj", "model.layers.53.mlp.experts.96.gate_proj", "model.layers.53.mlp.experts.97.gate_proj", "model.layers.53.mlp.experts.98.gate_proj", "model.layers.53.mlp.experts.99.gate_proj", "model.layers.53.mlp.experts.100.gate_proj", "model.layers.53.mlp.experts.101.gate_proj", "model.layers.53.mlp.experts.102.gate_proj", "model.layers.53.mlp.experts.103.gate_proj", "model.layers.53.mlp.experts.104.gate_proj", "model.layers.53.mlp.experts.105.gate_proj", "model.layers.53.mlp.experts.106.gate_proj", "model.layers.53.mlp.experts.107.gate_proj", "model.layers.53.mlp.experts.108.gate_proj", "model.layers.53.mlp.experts.109.gate_proj", "model.layers.53.mlp.experts.110.gate_proj", "model.layers.53.mlp.experts.111.gate_proj", "model.layers.53.mlp.experts.112.gate_proj", "model.layers.53.mlp.experts.113.gate_proj", "model.layers.53.mlp.experts.114.gate_proj", "model.layers.53.mlp.experts.115.gate_proj", "model.layers.53.mlp.experts.116.gate_proj", "model.layers.53.mlp.experts.117.gate_proj", "model.layers.53.mlp.experts.118.gate_proj", "model.layers.53.mlp.experts.119.gate_proj", "model.layers.53.mlp.experts.120.gate_proj", "model.layers.53.mlp.experts.121.gate_proj", "model.layers.53.mlp.experts.122.gate_proj", "model.layers.53.mlp.experts.123.gate_proj", "model.layers.53.mlp.experts.124.gate_proj", "model.layers.53.mlp.experts.125.gate_proj", "model.layers.53.mlp.experts.126.gate_proj", "model.layers.53.mlp.experts.127.gate_proj", "model.layers.53.mlp.experts.128.gate_proj", "model.layers.53.mlp.experts.129.gate_proj", "model.layers.53.mlp.experts.130.gate_proj", "model.layers.53.mlp.experts.131.gate_proj", "model.layers.53.mlp.experts.132.gate_proj", "model.layers.53.mlp.experts.133.gate_proj", "model.layers.53.mlp.experts.134.gate_proj", "model.layers.53.mlp.experts.135.gate_proj", "model.layers.53.mlp.experts.136.gate_proj", "model.layers.53.mlp.experts.137.gate_proj", "model.layers.53.mlp.experts.138.gate_proj", "model.layers.53.mlp.experts.139.gate_proj", "model.layers.53.mlp.experts.140.gate_proj", "model.layers.53.mlp.experts.141.gate_proj", "model.layers.53.mlp.experts.142.gate_proj", "model.layers.53.mlp.experts.143.gate_proj", "model.layers.53.mlp.experts.144.gate_proj", "model.layers.53.mlp.experts.145.gate_proj", "model.layers.53.mlp.experts.146.gate_proj", "model.layers.53.mlp.experts.147.gate_proj", "model.layers.53.mlp.experts.148.gate_proj", "model.layers.53.mlp.experts.149.gate_proj", "model.layers.53.mlp.experts.150.gate_proj", "model.layers.53.mlp.experts.151.gate_proj", "model.layers.53.mlp.experts.152.gate_proj", "model.layers.53.mlp.experts.153.gate_proj", "model.layers.53.mlp.experts.154.gate_proj", "model.layers.53.mlp.experts.155.gate_proj", "model.layers.53.mlp.experts.156.gate_proj", "model.layers.53.mlp.experts.157.gate_proj", "model.layers.53.mlp.experts.158.gate_proj", "model.layers.53.mlp.experts.159.gate_proj", "model.layers.53.mlp.experts.0.up_proj", "model.layers.53.mlp.experts.1.up_proj", "model.layers.53.mlp.experts.2.up_proj", "model.layers.53.mlp.experts.3.up_proj", "model.layers.53.mlp.experts.4.up_proj", "model.layers.53.mlp.experts.5.up_proj", "model.layers.53.mlp.experts.6.up_proj", "model.layers.53.mlp.experts.7.up_proj", "model.layers.53.mlp.experts.8.up_proj", "model.layers.53.mlp.experts.9.up_proj", "model.layers.53.mlp.experts.10.up_proj", "model.layers.53.mlp.experts.11.up_proj", "model.layers.53.mlp.experts.12.up_proj", "model.layers.53.mlp.experts.13.up_proj", "model.layers.53.mlp.experts.14.up_proj", "model.layers.53.mlp.experts.15.up_proj", "model.layers.53.mlp.experts.16.up_proj", "model.layers.53.mlp.experts.17.up_proj", "model.layers.53.mlp.experts.18.up_proj", "model.layers.53.mlp.experts.19.up_proj", "model.layers.53.mlp.experts.20.up_proj", "model.layers.53.mlp.experts.21.up_proj", "model.layers.53.mlp.experts.22.up_proj", "model.layers.53.mlp.experts.23.up_proj", "model.layers.53.mlp.experts.24.up_proj", "model.layers.53.mlp.experts.25.up_proj", "model.layers.53.mlp.experts.26.up_proj", "model.layers.53.mlp.experts.27.up_proj", "model.layers.53.mlp.experts.28.up_proj", "model.layers.53.mlp.experts.29.up_proj", "model.layers.53.mlp.experts.30.up_proj", "model.layers.53.mlp.experts.31.up_proj", "model.layers.53.mlp.experts.32.up_proj", "model.layers.53.mlp.experts.33.up_proj", "model.layers.53.mlp.experts.34.up_proj", "model.layers.53.mlp.experts.35.up_proj", "model.layers.53.mlp.experts.36.up_proj", "model.layers.53.mlp.experts.37.up_proj", "model.layers.53.mlp.experts.38.up_proj", "model.layers.53.mlp.experts.39.up_proj", "model.layers.53.mlp.experts.40.up_proj", "model.layers.53.mlp.experts.41.up_proj", "model.layers.53.mlp.experts.42.up_proj", "model.layers.53.mlp.experts.43.up_proj", "model.layers.53.mlp.experts.44.up_proj", "model.layers.53.mlp.experts.45.up_proj", "model.layers.53.mlp.experts.46.up_proj", "model.layers.53.mlp.experts.47.up_proj", "model.layers.53.mlp.experts.48.up_proj", "model.layers.53.mlp.experts.49.up_proj", "model.layers.53.mlp.experts.50.up_proj", "model.layers.53.mlp.experts.51.up_proj", "model.layers.53.mlp.experts.52.up_proj", "model.layers.53.mlp.experts.53.up_proj", "model.layers.53.mlp.experts.54.up_proj", "model.layers.53.mlp.experts.55.up_proj", "model.layers.53.mlp.experts.56.up_proj", "model.layers.53.mlp.experts.57.up_proj", "model.layers.53.mlp.experts.58.up_proj", "model.layers.53.mlp.experts.59.up_proj", "model.layers.53.mlp.experts.60.up_proj", "model.layers.53.mlp.experts.61.up_proj", "model.layers.53.mlp.experts.62.up_proj", "model.layers.53.mlp.experts.63.up_proj", "model.layers.53.mlp.experts.64.up_proj", "model.layers.53.mlp.experts.65.up_proj", "model.layers.53.mlp.experts.66.up_proj", "model.layers.53.mlp.experts.67.up_proj", "model.layers.53.mlp.experts.68.up_proj", "model.layers.53.mlp.experts.69.up_proj", "model.layers.53.mlp.experts.70.up_proj", "model.layers.53.mlp.experts.71.up_proj", "model.layers.53.mlp.experts.72.up_proj", "model.layers.53.mlp.experts.73.up_proj", "model.layers.53.mlp.experts.74.up_proj", "model.layers.53.mlp.experts.75.up_proj", "model.layers.53.mlp.experts.76.up_proj", "model.layers.53.mlp.experts.77.up_proj", "model.layers.53.mlp.experts.78.up_proj", "model.layers.53.mlp.experts.79.up_proj", "model.layers.53.mlp.experts.80.up_proj", "model.layers.53.mlp.experts.81.up_proj", "model.layers.53.mlp.experts.82.up_proj", "model.layers.53.mlp.experts.83.up_proj", "model.layers.53.mlp.experts.84.up_proj", "model.layers.53.mlp.experts.85.up_proj", "model.layers.53.mlp.experts.86.up_proj", "model.layers.53.mlp.experts.87.up_proj", "model.layers.53.mlp.experts.88.up_proj", "model.layers.53.mlp.experts.89.up_proj", "model.layers.53.mlp.experts.90.up_proj", "model.layers.53.mlp.experts.91.up_proj", "model.layers.53.mlp.experts.92.up_proj", "model.layers.53.mlp.experts.93.up_proj", "model.layers.53.mlp.experts.94.up_proj", "model.layers.53.mlp.experts.95.up_proj", "model.layers.53.mlp.experts.96.up_proj", "model.layers.53.mlp.experts.97.up_proj", "model.layers.53.mlp.experts.98.up_proj", "model.layers.53.mlp.experts.99.up_proj", "model.layers.53.mlp.experts.100.up_proj", "model.layers.53.mlp.experts.101.up_proj", "model.layers.53.mlp.experts.102.up_proj", "model.layers.53.mlp.experts.103.up_proj", "model.layers.53.mlp.experts.104.up_proj", "model.layers.53.mlp.experts.105.up_proj", "model.layers.53.mlp.experts.106.up_proj", "model.layers.53.mlp.experts.107.up_proj", "model.layers.53.mlp.experts.108.up_proj", "model.layers.53.mlp.experts.109.up_proj", "model.layers.53.mlp.experts.110.up_proj", "model.layers.53.mlp.experts.111.up_proj", "model.layers.53.mlp.experts.112.up_proj", "model.layers.53.mlp.experts.113.up_proj", "model.layers.53.mlp.experts.114.up_proj", "model.layers.53.mlp.experts.115.up_proj", "model.layers.53.mlp.experts.116.up_proj", "model.layers.53.mlp.experts.117.up_proj", "model.layers.53.mlp.experts.118.up_proj", "model.layers.53.mlp.experts.119.up_proj", "model.layers.53.mlp.experts.120.up_proj", "model.layers.53.mlp.experts.121.up_proj", "model.layers.53.mlp.experts.122.up_proj", "model.layers.53.mlp.experts.123.up_proj", "model.layers.53.mlp.experts.124.up_proj", "model.layers.53.mlp.experts.125.up_proj", "model.layers.53.mlp.experts.126.up_proj", "model.layers.53.mlp.experts.127.up_proj", "model.layers.53.mlp.experts.128.up_proj", "model.layers.53.mlp.experts.129.up_proj", "model.layers.53.mlp.experts.130.up_proj", "model.layers.53.mlp.experts.131.up_proj", "model.layers.53.mlp.experts.132.up_proj", "model.layers.53.mlp.experts.133.up_proj", "model.layers.53.mlp.experts.134.up_proj", "model.layers.53.mlp.experts.135.up_proj", "model.layers.53.mlp.experts.136.up_proj", "model.layers.53.mlp.experts.137.up_proj", "model.layers.53.mlp.experts.138.up_proj", "model.layers.53.mlp.experts.139.up_proj", "model.layers.53.mlp.experts.140.up_proj", "model.layers.53.mlp.experts.141.up_proj", "model.layers.53.mlp.experts.142.up_proj", "model.layers.53.mlp.experts.143.up_proj", "model.layers.53.mlp.experts.144.up_proj", "model.layers.53.mlp.experts.145.up_proj", "model.layers.53.mlp.experts.146.up_proj", "model.layers.53.mlp.experts.147.up_proj", "model.layers.53.mlp.experts.148.up_proj", "model.layers.53.mlp.experts.149.up_proj", "model.layers.53.mlp.experts.150.up_proj", "model.layers.53.mlp.experts.151.up_proj", "model.layers.53.mlp.experts.152.up_proj", "model.layers.53.mlp.experts.153.up_proj", "model.layers.53.mlp.experts.154.up_proj", "model.layers.53.mlp.experts.155.up_proj", "model.layers.53.mlp.experts.156.up_proj", "model.layers.53.mlp.experts.157.up_proj", "model.layers.53.mlp.experts.158.up_proj", "model.layers.53.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.977664841339426e-07, "dbits": 5033164800 } ] }, { "idx": 320, "layers": [ "model.layers.53.mlp.experts.0.down_proj", "model.layers.53.mlp.experts.1.down_proj", "model.layers.53.mlp.experts.2.down_proj", "model.layers.53.mlp.experts.3.down_proj", "model.layers.53.mlp.experts.4.down_proj", "model.layers.53.mlp.experts.5.down_proj", "model.layers.53.mlp.experts.6.down_proj", "model.layers.53.mlp.experts.7.down_proj", "model.layers.53.mlp.experts.8.down_proj", "model.layers.53.mlp.experts.9.down_proj", "model.layers.53.mlp.experts.10.down_proj", "model.layers.53.mlp.experts.11.down_proj", "model.layers.53.mlp.experts.12.down_proj", "model.layers.53.mlp.experts.13.down_proj", "model.layers.53.mlp.experts.14.down_proj", "model.layers.53.mlp.experts.15.down_proj", "model.layers.53.mlp.experts.16.down_proj", "model.layers.53.mlp.experts.17.down_proj", "model.layers.53.mlp.experts.18.down_proj", "model.layers.53.mlp.experts.19.down_proj", "model.layers.53.mlp.experts.20.down_proj", "model.layers.53.mlp.experts.21.down_proj", "model.layers.53.mlp.experts.22.down_proj", "model.layers.53.mlp.experts.23.down_proj", "model.layers.53.mlp.experts.24.down_proj", "model.layers.53.mlp.experts.25.down_proj", "model.layers.53.mlp.experts.26.down_proj", "model.layers.53.mlp.experts.27.down_proj", "model.layers.53.mlp.experts.28.down_proj", "model.layers.53.mlp.experts.29.down_proj", "model.layers.53.mlp.experts.30.down_proj", "model.layers.53.mlp.experts.31.down_proj", "model.layers.53.mlp.experts.32.down_proj", "model.layers.53.mlp.experts.33.down_proj", "model.layers.53.mlp.experts.34.down_proj", "model.layers.53.mlp.experts.35.down_proj", "model.layers.53.mlp.experts.36.down_proj", "model.layers.53.mlp.experts.37.down_proj", "model.layers.53.mlp.experts.38.down_proj", "model.layers.53.mlp.experts.39.down_proj", "model.layers.53.mlp.experts.40.down_proj", "model.layers.53.mlp.experts.41.down_proj", "model.layers.53.mlp.experts.42.down_proj", "model.layers.53.mlp.experts.43.down_proj", "model.layers.53.mlp.experts.44.down_proj", "model.layers.53.mlp.experts.45.down_proj", "model.layers.53.mlp.experts.46.down_proj", "model.layers.53.mlp.experts.47.down_proj", "model.layers.53.mlp.experts.48.down_proj", "model.layers.53.mlp.experts.49.down_proj", "model.layers.53.mlp.experts.50.down_proj", "model.layers.53.mlp.experts.51.down_proj", "model.layers.53.mlp.experts.52.down_proj", "model.layers.53.mlp.experts.53.down_proj", "model.layers.53.mlp.experts.54.down_proj", "model.layers.53.mlp.experts.55.down_proj", "model.layers.53.mlp.experts.56.down_proj", "model.layers.53.mlp.experts.57.down_proj", "model.layers.53.mlp.experts.58.down_proj", "model.layers.53.mlp.experts.59.down_proj", "model.layers.53.mlp.experts.60.down_proj", "model.layers.53.mlp.experts.61.down_proj", "model.layers.53.mlp.experts.62.down_proj", "model.layers.53.mlp.experts.63.down_proj", "model.layers.53.mlp.experts.64.down_proj", "model.layers.53.mlp.experts.65.down_proj", "model.layers.53.mlp.experts.66.down_proj", "model.layers.53.mlp.experts.67.down_proj", "model.layers.53.mlp.experts.68.down_proj", "model.layers.53.mlp.experts.69.down_proj", "model.layers.53.mlp.experts.70.down_proj", "model.layers.53.mlp.experts.71.down_proj", "model.layers.53.mlp.experts.72.down_proj", "model.layers.53.mlp.experts.73.down_proj", "model.layers.53.mlp.experts.74.down_proj", "model.layers.53.mlp.experts.75.down_proj", "model.layers.53.mlp.experts.76.down_proj", "model.layers.53.mlp.experts.77.down_proj", "model.layers.53.mlp.experts.78.down_proj", "model.layers.53.mlp.experts.79.down_proj", "model.layers.53.mlp.experts.80.down_proj", "model.layers.53.mlp.experts.81.down_proj", "model.layers.53.mlp.experts.82.down_proj", "model.layers.53.mlp.experts.83.down_proj", "model.layers.53.mlp.experts.84.down_proj", "model.layers.53.mlp.experts.85.down_proj", "model.layers.53.mlp.experts.86.down_proj", "model.layers.53.mlp.experts.87.down_proj", "model.layers.53.mlp.experts.88.down_proj", "model.layers.53.mlp.experts.89.down_proj", "model.layers.53.mlp.experts.90.down_proj", "model.layers.53.mlp.experts.91.down_proj", "model.layers.53.mlp.experts.92.down_proj", "model.layers.53.mlp.experts.93.down_proj", "model.layers.53.mlp.experts.94.down_proj", "model.layers.53.mlp.experts.95.down_proj", "model.layers.53.mlp.experts.96.down_proj", "model.layers.53.mlp.experts.97.down_proj", "model.layers.53.mlp.experts.98.down_proj", "model.layers.53.mlp.experts.99.down_proj", "model.layers.53.mlp.experts.100.down_proj", "model.layers.53.mlp.experts.101.down_proj", "model.layers.53.mlp.experts.102.down_proj", "model.layers.53.mlp.experts.103.down_proj", "model.layers.53.mlp.experts.104.down_proj", "model.layers.53.mlp.experts.105.down_proj", "model.layers.53.mlp.experts.106.down_proj", "model.layers.53.mlp.experts.107.down_proj", "model.layers.53.mlp.experts.108.down_proj", "model.layers.53.mlp.experts.109.down_proj", "model.layers.53.mlp.experts.110.down_proj", "model.layers.53.mlp.experts.111.down_proj", "model.layers.53.mlp.experts.112.down_proj", "model.layers.53.mlp.experts.113.down_proj", "model.layers.53.mlp.experts.114.down_proj", "model.layers.53.mlp.experts.115.down_proj", "model.layers.53.mlp.experts.116.down_proj", "model.layers.53.mlp.experts.117.down_proj", "model.layers.53.mlp.experts.118.down_proj", "model.layers.53.mlp.experts.119.down_proj", "model.layers.53.mlp.experts.120.down_proj", "model.layers.53.mlp.experts.121.down_proj", "model.layers.53.mlp.experts.122.down_proj", "model.layers.53.mlp.experts.123.down_proj", "model.layers.53.mlp.experts.124.down_proj", "model.layers.53.mlp.experts.125.down_proj", "model.layers.53.mlp.experts.126.down_proj", "model.layers.53.mlp.experts.127.down_proj", "model.layers.53.mlp.experts.128.down_proj", "model.layers.53.mlp.experts.129.down_proj", "model.layers.53.mlp.experts.130.down_proj", "model.layers.53.mlp.experts.131.down_proj", "model.layers.53.mlp.experts.132.down_proj", "model.layers.53.mlp.experts.133.down_proj", "model.layers.53.mlp.experts.134.down_proj", "model.layers.53.mlp.experts.135.down_proj", "model.layers.53.mlp.experts.136.down_proj", "model.layers.53.mlp.experts.137.down_proj", "model.layers.53.mlp.experts.138.down_proj", "model.layers.53.mlp.experts.139.down_proj", "model.layers.53.mlp.experts.140.down_proj", "model.layers.53.mlp.experts.141.down_proj", "model.layers.53.mlp.experts.142.down_proj", "model.layers.53.mlp.experts.143.down_proj", "model.layers.53.mlp.experts.144.down_proj", "model.layers.53.mlp.experts.145.down_proj", "model.layers.53.mlp.experts.146.down_proj", "model.layers.53.mlp.experts.147.down_proj", "model.layers.53.mlp.experts.148.down_proj", "model.layers.53.mlp.experts.149.down_proj", "model.layers.53.mlp.experts.150.down_proj", "model.layers.53.mlp.experts.151.down_proj", "model.layers.53.mlp.experts.152.down_proj", "model.layers.53.mlp.experts.153.down_proj", "model.layers.53.mlp.experts.154.down_proj", "model.layers.53.mlp.experts.155.down_proj", "model.layers.53.mlp.experts.156.down_proj", "model.layers.53.mlp.experts.157.down_proj", "model.layers.53.mlp.experts.158.down_proj", "model.layers.53.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.942502710036977e-06, "dbits": 2516582400 } ] }, { "idx": 321, "layers": [ "model.layers.54.self_attn.q_proj" ], "candidates": [ { "dkld": 4.106125561520414e-06, "dbits": 125829120 } ] }, { "idx": 322, "layers": [ "model.layers.54.self_attn.k_proj", "model.layers.54.self_attn.v_proj" ], "candidates": [ { "dkld": 2.389616565778895e-06, "dbits": 20971520 } ] }, { "idx": 323, "layers": [ "model.layers.54.self_attn.o_proj" ], "candidates": [ { "dkld": 1.5333411283790848e-06, "dbits": 125829120 } ] }, { "idx": 324, "layers": [ "model.layers.54.mlp.shared_experts.gate_proj", "model.layers.54.mlp.shared_experts.up_proj", "model.layers.54.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.4674303606109166e-08, "dbits": 47185920 } ] }, { "idx": 325, "layers": [ "model.layers.54.mlp.experts.0.gate_proj", "model.layers.54.mlp.experts.1.gate_proj", "model.layers.54.mlp.experts.2.gate_proj", "model.layers.54.mlp.experts.3.gate_proj", "model.layers.54.mlp.experts.4.gate_proj", "model.layers.54.mlp.experts.5.gate_proj", "model.layers.54.mlp.experts.6.gate_proj", "model.layers.54.mlp.experts.7.gate_proj", "model.layers.54.mlp.experts.8.gate_proj", "model.layers.54.mlp.experts.9.gate_proj", "model.layers.54.mlp.experts.10.gate_proj", "model.layers.54.mlp.experts.11.gate_proj", "model.layers.54.mlp.experts.12.gate_proj", "model.layers.54.mlp.experts.13.gate_proj", "model.layers.54.mlp.experts.14.gate_proj", "model.layers.54.mlp.experts.15.gate_proj", "model.layers.54.mlp.experts.16.gate_proj", "model.layers.54.mlp.experts.17.gate_proj", "model.layers.54.mlp.experts.18.gate_proj", "model.layers.54.mlp.experts.19.gate_proj", "model.layers.54.mlp.experts.20.gate_proj", "model.layers.54.mlp.experts.21.gate_proj", "model.layers.54.mlp.experts.22.gate_proj", "model.layers.54.mlp.experts.23.gate_proj", "model.layers.54.mlp.experts.24.gate_proj", "model.layers.54.mlp.experts.25.gate_proj", "model.layers.54.mlp.experts.26.gate_proj", "model.layers.54.mlp.experts.27.gate_proj", "model.layers.54.mlp.experts.28.gate_proj", "model.layers.54.mlp.experts.29.gate_proj", "model.layers.54.mlp.experts.30.gate_proj", "model.layers.54.mlp.experts.31.gate_proj", "model.layers.54.mlp.experts.32.gate_proj", "model.layers.54.mlp.experts.33.gate_proj", "model.layers.54.mlp.experts.34.gate_proj", "model.layers.54.mlp.experts.35.gate_proj", "model.layers.54.mlp.experts.36.gate_proj", "model.layers.54.mlp.experts.37.gate_proj", "model.layers.54.mlp.experts.38.gate_proj", "model.layers.54.mlp.experts.39.gate_proj", "model.layers.54.mlp.experts.40.gate_proj", "model.layers.54.mlp.experts.41.gate_proj", "model.layers.54.mlp.experts.42.gate_proj", "model.layers.54.mlp.experts.43.gate_proj", "model.layers.54.mlp.experts.44.gate_proj", "model.layers.54.mlp.experts.45.gate_proj", "model.layers.54.mlp.experts.46.gate_proj", "model.layers.54.mlp.experts.47.gate_proj", "model.layers.54.mlp.experts.48.gate_proj", "model.layers.54.mlp.experts.49.gate_proj", "model.layers.54.mlp.experts.50.gate_proj", "model.layers.54.mlp.experts.51.gate_proj", "model.layers.54.mlp.experts.52.gate_proj", "model.layers.54.mlp.experts.53.gate_proj", "model.layers.54.mlp.experts.54.gate_proj", "model.layers.54.mlp.experts.55.gate_proj", "model.layers.54.mlp.experts.56.gate_proj", "model.layers.54.mlp.experts.57.gate_proj", "model.layers.54.mlp.experts.58.gate_proj", "model.layers.54.mlp.experts.59.gate_proj", "model.layers.54.mlp.experts.60.gate_proj", "model.layers.54.mlp.experts.61.gate_proj", "model.layers.54.mlp.experts.62.gate_proj", "model.layers.54.mlp.experts.63.gate_proj", "model.layers.54.mlp.experts.64.gate_proj", "model.layers.54.mlp.experts.65.gate_proj", "model.layers.54.mlp.experts.66.gate_proj", "model.layers.54.mlp.experts.67.gate_proj", "model.layers.54.mlp.experts.68.gate_proj", "model.layers.54.mlp.experts.69.gate_proj", "model.layers.54.mlp.experts.70.gate_proj", "model.layers.54.mlp.experts.71.gate_proj", "model.layers.54.mlp.experts.72.gate_proj", "model.layers.54.mlp.experts.73.gate_proj", "model.layers.54.mlp.experts.74.gate_proj", "model.layers.54.mlp.experts.75.gate_proj", "model.layers.54.mlp.experts.76.gate_proj", "model.layers.54.mlp.experts.77.gate_proj", "model.layers.54.mlp.experts.78.gate_proj", "model.layers.54.mlp.experts.79.gate_proj", "model.layers.54.mlp.experts.80.gate_proj", "model.layers.54.mlp.experts.81.gate_proj", "model.layers.54.mlp.experts.82.gate_proj", "model.layers.54.mlp.experts.83.gate_proj", "model.layers.54.mlp.experts.84.gate_proj", "model.layers.54.mlp.experts.85.gate_proj", "model.layers.54.mlp.experts.86.gate_proj", "model.layers.54.mlp.experts.87.gate_proj", "model.layers.54.mlp.experts.88.gate_proj", "model.layers.54.mlp.experts.89.gate_proj", "model.layers.54.mlp.experts.90.gate_proj", "model.layers.54.mlp.experts.91.gate_proj", "model.layers.54.mlp.experts.92.gate_proj", "model.layers.54.mlp.experts.93.gate_proj", "model.layers.54.mlp.experts.94.gate_proj", "model.layers.54.mlp.experts.95.gate_proj", "model.layers.54.mlp.experts.96.gate_proj", "model.layers.54.mlp.experts.97.gate_proj", "model.layers.54.mlp.experts.98.gate_proj", "model.layers.54.mlp.experts.99.gate_proj", "model.layers.54.mlp.experts.100.gate_proj", "model.layers.54.mlp.experts.101.gate_proj", "model.layers.54.mlp.experts.102.gate_proj", "model.layers.54.mlp.experts.103.gate_proj", "model.layers.54.mlp.experts.104.gate_proj", "model.layers.54.mlp.experts.105.gate_proj", "model.layers.54.mlp.experts.106.gate_proj", "model.layers.54.mlp.experts.107.gate_proj", "model.layers.54.mlp.experts.108.gate_proj", "model.layers.54.mlp.experts.109.gate_proj", "model.layers.54.mlp.experts.110.gate_proj", "model.layers.54.mlp.experts.111.gate_proj", "model.layers.54.mlp.experts.112.gate_proj", "model.layers.54.mlp.experts.113.gate_proj", "model.layers.54.mlp.experts.114.gate_proj", "model.layers.54.mlp.experts.115.gate_proj", "model.layers.54.mlp.experts.116.gate_proj", "model.layers.54.mlp.experts.117.gate_proj", "model.layers.54.mlp.experts.118.gate_proj", "model.layers.54.mlp.experts.119.gate_proj", "model.layers.54.mlp.experts.120.gate_proj", "model.layers.54.mlp.experts.121.gate_proj", "model.layers.54.mlp.experts.122.gate_proj", "model.layers.54.mlp.experts.123.gate_proj", "model.layers.54.mlp.experts.124.gate_proj", "model.layers.54.mlp.experts.125.gate_proj", "model.layers.54.mlp.experts.126.gate_proj", "model.layers.54.mlp.experts.127.gate_proj", "model.layers.54.mlp.experts.128.gate_proj", "model.layers.54.mlp.experts.129.gate_proj", "model.layers.54.mlp.experts.130.gate_proj", "model.layers.54.mlp.experts.131.gate_proj", "model.layers.54.mlp.experts.132.gate_proj", "model.layers.54.mlp.experts.133.gate_proj", "model.layers.54.mlp.experts.134.gate_proj", "model.layers.54.mlp.experts.135.gate_proj", "model.layers.54.mlp.experts.136.gate_proj", "model.layers.54.mlp.experts.137.gate_proj", "model.layers.54.mlp.experts.138.gate_proj", "model.layers.54.mlp.experts.139.gate_proj", "model.layers.54.mlp.experts.140.gate_proj", "model.layers.54.mlp.experts.141.gate_proj", "model.layers.54.mlp.experts.142.gate_proj", "model.layers.54.mlp.experts.143.gate_proj", "model.layers.54.mlp.experts.144.gate_proj", "model.layers.54.mlp.experts.145.gate_proj", "model.layers.54.mlp.experts.146.gate_proj", "model.layers.54.mlp.experts.147.gate_proj", "model.layers.54.mlp.experts.148.gate_proj", "model.layers.54.mlp.experts.149.gate_proj", "model.layers.54.mlp.experts.150.gate_proj", "model.layers.54.mlp.experts.151.gate_proj", "model.layers.54.mlp.experts.152.gate_proj", "model.layers.54.mlp.experts.153.gate_proj", "model.layers.54.mlp.experts.154.gate_proj", "model.layers.54.mlp.experts.155.gate_proj", "model.layers.54.mlp.experts.156.gate_proj", "model.layers.54.mlp.experts.157.gate_proj", "model.layers.54.mlp.experts.158.gate_proj", "model.layers.54.mlp.experts.159.gate_proj", "model.layers.54.mlp.experts.0.up_proj", "model.layers.54.mlp.experts.1.up_proj", "model.layers.54.mlp.experts.2.up_proj", "model.layers.54.mlp.experts.3.up_proj", "model.layers.54.mlp.experts.4.up_proj", "model.layers.54.mlp.experts.5.up_proj", "model.layers.54.mlp.experts.6.up_proj", "model.layers.54.mlp.experts.7.up_proj", "model.layers.54.mlp.experts.8.up_proj", "model.layers.54.mlp.experts.9.up_proj", "model.layers.54.mlp.experts.10.up_proj", "model.layers.54.mlp.experts.11.up_proj", "model.layers.54.mlp.experts.12.up_proj", "model.layers.54.mlp.experts.13.up_proj", "model.layers.54.mlp.experts.14.up_proj", "model.layers.54.mlp.experts.15.up_proj", "model.layers.54.mlp.experts.16.up_proj", "model.layers.54.mlp.experts.17.up_proj", "model.layers.54.mlp.experts.18.up_proj", "model.layers.54.mlp.experts.19.up_proj", "model.layers.54.mlp.experts.20.up_proj", "model.layers.54.mlp.experts.21.up_proj", "model.layers.54.mlp.experts.22.up_proj", "model.layers.54.mlp.experts.23.up_proj", "model.layers.54.mlp.experts.24.up_proj", "model.layers.54.mlp.experts.25.up_proj", "model.layers.54.mlp.experts.26.up_proj", "model.layers.54.mlp.experts.27.up_proj", "model.layers.54.mlp.experts.28.up_proj", "model.layers.54.mlp.experts.29.up_proj", "model.layers.54.mlp.experts.30.up_proj", "model.layers.54.mlp.experts.31.up_proj", "model.layers.54.mlp.experts.32.up_proj", "model.layers.54.mlp.experts.33.up_proj", "model.layers.54.mlp.experts.34.up_proj", "model.layers.54.mlp.experts.35.up_proj", "model.layers.54.mlp.experts.36.up_proj", "model.layers.54.mlp.experts.37.up_proj", "model.layers.54.mlp.experts.38.up_proj", "model.layers.54.mlp.experts.39.up_proj", "model.layers.54.mlp.experts.40.up_proj", "model.layers.54.mlp.experts.41.up_proj", "model.layers.54.mlp.experts.42.up_proj", "model.layers.54.mlp.experts.43.up_proj", "model.layers.54.mlp.experts.44.up_proj", "model.layers.54.mlp.experts.45.up_proj", "model.layers.54.mlp.experts.46.up_proj", "model.layers.54.mlp.experts.47.up_proj", "model.layers.54.mlp.experts.48.up_proj", "model.layers.54.mlp.experts.49.up_proj", "model.layers.54.mlp.experts.50.up_proj", "model.layers.54.mlp.experts.51.up_proj", "model.layers.54.mlp.experts.52.up_proj", "model.layers.54.mlp.experts.53.up_proj", "model.layers.54.mlp.experts.54.up_proj", "model.layers.54.mlp.experts.55.up_proj", "model.layers.54.mlp.experts.56.up_proj", "model.layers.54.mlp.experts.57.up_proj", "model.layers.54.mlp.experts.58.up_proj", "model.layers.54.mlp.experts.59.up_proj", "model.layers.54.mlp.experts.60.up_proj", "model.layers.54.mlp.experts.61.up_proj", "model.layers.54.mlp.experts.62.up_proj", "model.layers.54.mlp.experts.63.up_proj", "model.layers.54.mlp.experts.64.up_proj", "model.layers.54.mlp.experts.65.up_proj", "model.layers.54.mlp.experts.66.up_proj", "model.layers.54.mlp.experts.67.up_proj", "model.layers.54.mlp.experts.68.up_proj", "model.layers.54.mlp.experts.69.up_proj", "model.layers.54.mlp.experts.70.up_proj", "model.layers.54.mlp.experts.71.up_proj", "model.layers.54.mlp.experts.72.up_proj", "model.layers.54.mlp.experts.73.up_proj", "model.layers.54.mlp.experts.74.up_proj", "model.layers.54.mlp.experts.75.up_proj", "model.layers.54.mlp.experts.76.up_proj", "model.layers.54.mlp.experts.77.up_proj", "model.layers.54.mlp.experts.78.up_proj", "model.layers.54.mlp.experts.79.up_proj", "model.layers.54.mlp.experts.80.up_proj", "model.layers.54.mlp.experts.81.up_proj", "model.layers.54.mlp.experts.82.up_proj", "model.layers.54.mlp.experts.83.up_proj", "model.layers.54.mlp.experts.84.up_proj", "model.layers.54.mlp.experts.85.up_proj", "model.layers.54.mlp.experts.86.up_proj", "model.layers.54.mlp.experts.87.up_proj", "model.layers.54.mlp.experts.88.up_proj", "model.layers.54.mlp.experts.89.up_proj", "model.layers.54.mlp.experts.90.up_proj", "model.layers.54.mlp.experts.91.up_proj", "model.layers.54.mlp.experts.92.up_proj", "model.layers.54.mlp.experts.93.up_proj", "model.layers.54.mlp.experts.94.up_proj", "model.layers.54.mlp.experts.95.up_proj", "model.layers.54.mlp.experts.96.up_proj", "model.layers.54.mlp.experts.97.up_proj", "model.layers.54.mlp.experts.98.up_proj", "model.layers.54.mlp.experts.99.up_proj", "model.layers.54.mlp.experts.100.up_proj", "model.layers.54.mlp.experts.101.up_proj", "model.layers.54.mlp.experts.102.up_proj", "model.layers.54.mlp.experts.103.up_proj", "model.layers.54.mlp.experts.104.up_proj", "model.layers.54.mlp.experts.105.up_proj", "model.layers.54.mlp.experts.106.up_proj", "model.layers.54.mlp.experts.107.up_proj", "model.layers.54.mlp.experts.108.up_proj", "model.layers.54.mlp.experts.109.up_proj", "model.layers.54.mlp.experts.110.up_proj", "model.layers.54.mlp.experts.111.up_proj", "model.layers.54.mlp.experts.112.up_proj", "model.layers.54.mlp.experts.113.up_proj", "model.layers.54.mlp.experts.114.up_proj", "model.layers.54.mlp.experts.115.up_proj", "model.layers.54.mlp.experts.116.up_proj", "model.layers.54.mlp.experts.117.up_proj", "model.layers.54.mlp.experts.118.up_proj", "model.layers.54.mlp.experts.119.up_proj", "model.layers.54.mlp.experts.120.up_proj", "model.layers.54.mlp.experts.121.up_proj", "model.layers.54.mlp.experts.122.up_proj", "model.layers.54.mlp.experts.123.up_proj", "model.layers.54.mlp.experts.124.up_proj", "model.layers.54.mlp.experts.125.up_proj", "model.layers.54.mlp.experts.126.up_proj", "model.layers.54.mlp.experts.127.up_proj", "model.layers.54.mlp.experts.128.up_proj", "model.layers.54.mlp.experts.129.up_proj", "model.layers.54.mlp.experts.130.up_proj", "model.layers.54.mlp.experts.131.up_proj", "model.layers.54.mlp.experts.132.up_proj", "model.layers.54.mlp.experts.133.up_proj", "model.layers.54.mlp.experts.134.up_proj", "model.layers.54.mlp.experts.135.up_proj", "model.layers.54.mlp.experts.136.up_proj", "model.layers.54.mlp.experts.137.up_proj", "model.layers.54.mlp.experts.138.up_proj", "model.layers.54.mlp.experts.139.up_proj", "model.layers.54.mlp.experts.140.up_proj", "model.layers.54.mlp.experts.141.up_proj", "model.layers.54.mlp.experts.142.up_proj", "model.layers.54.mlp.experts.143.up_proj", "model.layers.54.mlp.experts.144.up_proj", "model.layers.54.mlp.experts.145.up_proj", "model.layers.54.mlp.experts.146.up_proj", "model.layers.54.mlp.experts.147.up_proj", "model.layers.54.mlp.experts.148.up_proj", "model.layers.54.mlp.experts.149.up_proj", "model.layers.54.mlp.experts.150.up_proj", "model.layers.54.mlp.experts.151.up_proj", "model.layers.54.mlp.experts.152.up_proj", "model.layers.54.mlp.experts.153.up_proj", "model.layers.54.mlp.experts.154.up_proj", "model.layers.54.mlp.experts.155.up_proj", "model.layers.54.mlp.experts.156.up_proj", "model.layers.54.mlp.experts.157.up_proj", "model.layers.54.mlp.experts.158.up_proj", "model.layers.54.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -4.197296220809438e-07, "dbits": 5033164800 } ] }, { "idx": 326, "layers": [ "model.layers.54.mlp.experts.0.down_proj", "model.layers.54.mlp.experts.1.down_proj", "model.layers.54.mlp.experts.2.down_proj", "model.layers.54.mlp.experts.3.down_proj", "model.layers.54.mlp.experts.4.down_proj", "model.layers.54.mlp.experts.5.down_proj", "model.layers.54.mlp.experts.6.down_proj", "model.layers.54.mlp.experts.7.down_proj", "model.layers.54.mlp.experts.8.down_proj", "model.layers.54.mlp.experts.9.down_proj", "model.layers.54.mlp.experts.10.down_proj", "model.layers.54.mlp.experts.11.down_proj", "model.layers.54.mlp.experts.12.down_proj", "model.layers.54.mlp.experts.13.down_proj", "model.layers.54.mlp.experts.14.down_proj", "model.layers.54.mlp.experts.15.down_proj", "model.layers.54.mlp.experts.16.down_proj", "model.layers.54.mlp.experts.17.down_proj", "model.layers.54.mlp.experts.18.down_proj", "model.layers.54.mlp.experts.19.down_proj", "model.layers.54.mlp.experts.20.down_proj", "model.layers.54.mlp.experts.21.down_proj", "model.layers.54.mlp.experts.22.down_proj", "model.layers.54.mlp.experts.23.down_proj", "model.layers.54.mlp.experts.24.down_proj", "model.layers.54.mlp.experts.25.down_proj", "model.layers.54.mlp.experts.26.down_proj", "model.layers.54.mlp.experts.27.down_proj", "model.layers.54.mlp.experts.28.down_proj", "model.layers.54.mlp.experts.29.down_proj", "model.layers.54.mlp.experts.30.down_proj", "model.layers.54.mlp.experts.31.down_proj", "model.layers.54.mlp.experts.32.down_proj", "model.layers.54.mlp.experts.33.down_proj", "model.layers.54.mlp.experts.34.down_proj", "model.layers.54.mlp.experts.35.down_proj", "model.layers.54.mlp.experts.36.down_proj", "model.layers.54.mlp.experts.37.down_proj", "model.layers.54.mlp.experts.38.down_proj", "model.layers.54.mlp.experts.39.down_proj", "model.layers.54.mlp.experts.40.down_proj", "model.layers.54.mlp.experts.41.down_proj", "model.layers.54.mlp.experts.42.down_proj", "model.layers.54.mlp.experts.43.down_proj", "model.layers.54.mlp.experts.44.down_proj", "model.layers.54.mlp.experts.45.down_proj", "model.layers.54.mlp.experts.46.down_proj", "model.layers.54.mlp.experts.47.down_proj", "model.layers.54.mlp.experts.48.down_proj", "model.layers.54.mlp.experts.49.down_proj", "model.layers.54.mlp.experts.50.down_proj", "model.layers.54.mlp.experts.51.down_proj", "model.layers.54.mlp.experts.52.down_proj", "model.layers.54.mlp.experts.53.down_proj", "model.layers.54.mlp.experts.54.down_proj", "model.layers.54.mlp.experts.55.down_proj", "model.layers.54.mlp.experts.56.down_proj", "model.layers.54.mlp.experts.57.down_proj", "model.layers.54.mlp.experts.58.down_proj", "model.layers.54.mlp.experts.59.down_proj", "model.layers.54.mlp.experts.60.down_proj", "model.layers.54.mlp.experts.61.down_proj", "model.layers.54.mlp.experts.62.down_proj", "model.layers.54.mlp.experts.63.down_proj", "model.layers.54.mlp.experts.64.down_proj", "model.layers.54.mlp.experts.65.down_proj", "model.layers.54.mlp.experts.66.down_proj", "model.layers.54.mlp.experts.67.down_proj", "model.layers.54.mlp.experts.68.down_proj", "model.layers.54.mlp.experts.69.down_proj", "model.layers.54.mlp.experts.70.down_proj", "model.layers.54.mlp.experts.71.down_proj", "model.layers.54.mlp.experts.72.down_proj", "model.layers.54.mlp.experts.73.down_proj", "model.layers.54.mlp.experts.74.down_proj", "model.layers.54.mlp.experts.75.down_proj", "model.layers.54.mlp.experts.76.down_proj", "model.layers.54.mlp.experts.77.down_proj", "model.layers.54.mlp.experts.78.down_proj", "model.layers.54.mlp.experts.79.down_proj", "model.layers.54.mlp.experts.80.down_proj", "model.layers.54.mlp.experts.81.down_proj", "model.layers.54.mlp.experts.82.down_proj", "model.layers.54.mlp.experts.83.down_proj", "model.layers.54.mlp.experts.84.down_proj", "model.layers.54.mlp.experts.85.down_proj", "model.layers.54.mlp.experts.86.down_proj", "model.layers.54.mlp.experts.87.down_proj", "model.layers.54.mlp.experts.88.down_proj", "model.layers.54.mlp.experts.89.down_proj", "model.layers.54.mlp.experts.90.down_proj", "model.layers.54.mlp.experts.91.down_proj", "model.layers.54.mlp.experts.92.down_proj", "model.layers.54.mlp.experts.93.down_proj", "model.layers.54.mlp.experts.94.down_proj", "model.layers.54.mlp.experts.95.down_proj", "model.layers.54.mlp.experts.96.down_proj", "model.layers.54.mlp.experts.97.down_proj", "model.layers.54.mlp.experts.98.down_proj", "model.layers.54.mlp.experts.99.down_proj", "model.layers.54.mlp.experts.100.down_proj", "model.layers.54.mlp.experts.101.down_proj", "model.layers.54.mlp.experts.102.down_proj", "model.layers.54.mlp.experts.103.down_proj", "model.layers.54.mlp.experts.104.down_proj", "model.layers.54.mlp.experts.105.down_proj", "model.layers.54.mlp.experts.106.down_proj", "model.layers.54.mlp.experts.107.down_proj", "model.layers.54.mlp.experts.108.down_proj", "model.layers.54.mlp.experts.109.down_proj", "model.layers.54.mlp.experts.110.down_proj", "model.layers.54.mlp.experts.111.down_proj", "model.layers.54.mlp.experts.112.down_proj", "model.layers.54.mlp.experts.113.down_proj", "model.layers.54.mlp.experts.114.down_proj", "model.layers.54.mlp.experts.115.down_proj", "model.layers.54.mlp.experts.116.down_proj", "model.layers.54.mlp.experts.117.down_proj", "model.layers.54.mlp.experts.118.down_proj", "model.layers.54.mlp.experts.119.down_proj", "model.layers.54.mlp.experts.120.down_proj", "model.layers.54.mlp.experts.121.down_proj", "model.layers.54.mlp.experts.122.down_proj", "model.layers.54.mlp.experts.123.down_proj", "model.layers.54.mlp.experts.124.down_proj", "model.layers.54.mlp.experts.125.down_proj", "model.layers.54.mlp.experts.126.down_proj", "model.layers.54.mlp.experts.127.down_proj", "model.layers.54.mlp.experts.128.down_proj", "model.layers.54.mlp.experts.129.down_proj", "model.layers.54.mlp.experts.130.down_proj", "model.layers.54.mlp.experts.131.down_proj", "model.layers.54.mlp.experts.132.down_proj", "model.layers.54.mlp.experts.133.down_proj", "model.layers.54.mlp.experts.134.down_proj", "model.layers.54.mlp.experts.135.down_proj", "model.layers.54.mlp.experts.136.down_proj", "model.layers.54.mlp.experts.137.down_proj", "model.layers.54.mlp.experts.138.down_proj", "model.layers.54.mlp.experts.139.down_proj", "model.layers.54.mlp.experts.140.down_proj", "model.layers.54.mlp.experts.141.down_proj", "model.layers.54.mlp.experts.142.down_proj", "model.layers.54.mlp.experts.143.down_proj", "model.layers.54.mlp.experts.144.down_proj", "model.layers.54.mlp.experts.145.down_proj", "model.layers.54.mlp.experts.146.down_proj", "model.layers.54.mlp.experts.147.down_proj", "model.layers.54.mlp.experts.148.down_proj", "model.layers.54.mlp.experts.149.down_proj", "model.layers.54.mlp.experts.150.down_proj", "model.layers.54.mlp.experts.151.down_proj", "model.layers.54.mlp.experts.152.down_proj", "model.layers.54.mlp.experts.153.down_proj", "model.layers.54.mlp.experts.154.down_proj", "model.layers.54.mlp.experts.155.down_proj", "model.layers.54.mlp.experts.156.down_proj", "model.layers.54.mlp.experts.157.down_proj", "model.layers.54.mlp.experts.158.down_proj", "model.layers.54.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.8122391086071947e-06, "dbits": 2516582400 } ] }, { "idx": 327, "layers": [ "model.layers.55.self_attn.q_proj" ], "candidates": [ { "dkld": 3.5581062547862963e-06, "dbits": 125829120 } ] }, { "idx": 328, "layers": [ "model.layers.55.self_attn.k_proj", "model.layers.55.self_attn.v_proj" ], "candidates": [ { "dkld": -1.7647253116592765e-06, "dbits": 20971520 } ] }, { "idx": 329, "layers": [ "model.layers.55.self_attn.o_proj" ], "candidates": [ { "dkld": 2.526008756831505e-07, "dbits": 125829120 } ] }, { "idx": 330, "layers": [ "model.layers.55.mlp.shared_experts.gate_proj", "model.layers.55.mlp.shared_experts.up_proj", "model.layers.55.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.3940781829878254e-06, "dbits": 47185920 } ] }, { "idx": 331, "layers": [ "model.layers.55.mlp.experts.0.gate_proj", "model.layers.55.mlp.experts.1.gate_proj", "model.layers.55.mlp.experts.2.gate_proj", "model.layers.55.mlp.experts.3.gate_proj", "model.layers.55.mlp.experts.4.gate_proj", "model.layers.55.mlp.experts.5.gate_proj", "model.layers.55.mlp.experts.6.gate_proj", "model.layers.55.mlp.experts.7.gate_proj", "model.layers.55.mlp.experts.8.gate_proj", "model.layers.55.mlp.experts.9.gate_proj", "model.layers.55.mlp.experts.10.gate_proj", "model.layers.55.mlp.experts.11.gate_proj", "model.layers.55.mlp.experts.12.gate_proj", "model.layers.55.mlp.experts.13.gate_proj", "model.layers.55.mlp.experts.14.gate_proj", "model.layers.55.mlp.experts.15.gate_proj", "model.layers.55.mlp.experts.16.gate_proj", "model.layers.55.mlp.experts.17.gate_proj", "model.layers.55.mlp.experts.18.gate_proj", "model.layers.55.mlp.experts.19.gate_proj", "model.layers.55.mlp.experts.20.gate_proj", "model.layers.55.mlp.experts.21.gate_proj", "model.layers.55.mlp.experts.22.gate_proj", "model.layers.55.mlp.experts.23.gate_proj", "model.layers.55.mlp.experts.24.gate_proj", "model.layers.55.mlp.experts.25.gate_proj", "model.layers.55.mlp.experts.26.gate_proj", "model.layers.55.mlp.experts.27.gate_proj", "model.layers.55.mlp.experts.28.gate_proj", "model.layers.55.mlp.experts.29.gate_proj", "model.layers.55.mlp.experts.30.gate_proj", "model.layers.55.mlp.experts.31.gate_proj", "model.layers.55.mlp.experts.32.gate_proj", "model.layers.55.mlp.experts.33.gate_proj", "model.layers.55.mlp.experts.34.gate_proj", "model.layers.55.mlp.experts.35.gate_proj", "model.layers.55.mlp.experts.36.gate_proj", "model.layers.55.mlp.experts.37.gate_proj", "model.layers.55.mlp.experts.38.gate_proj", "model.layers.55.mlp.experts.39.gate_proj", "model.layers.55.mlp.experts.40.gate_proj", "model.layers.55.mlp.experts.41.gate_proj", "model.layers.55.mlp.experts.42.gate_proj", "model.layers.55.mlp.experts.43.gate_proj", "model.layers.55.mlp.experts.44.gate_proj", "model.layers.55.mlp.experts.45.gate_proj", "model.layers.55.mlp.experts.46.gate_proj", "model.layers.55.mlp.experts.47.gate_proj", "model.layers.55.mlp.experts.48.gate_proj", "model.layers.55.mlp.experts.49.gate_proj", "model.layers.55.mlp.experts.50.gate_proj", "model.layers.55.mlp.experts.51.gate_proj", "model.layers.55.mlp.experts.52.gate_proj", "model.layers.55.mlp.experts.53.gate_proj", "model.layers.55.mlp.experts.54.gate_proj", "model.layers.55.mlp.experts.55.gate_proj", "model.layers.55.mlp.experts.56.gate_proj", "model.layers.55.mlp.experts.57.gate_proj", "model.layers.55.mlp.experts.58.gate_proj", "model.layers.55.mlp.experts.59.gate_proj", "model.layers.55.mlp.experts.60.gate_proj", "model.layers.55.mlp.experts.61.gate_proj", "model.layers.55.mlp.experts.62.gate_proj", "model.layers.55.mlp.experts.63.gate_proj", "model.layers.55.mlp.experts.64.gate_proj", "model.layers.55.mlp.experts.65.gate_proj", "model.layers.55.mlp.experts.66.gate_proj", "model.layers.55.mlp.experts.67.gate_proj", "model.layers.55.mlp.experts.68.gate_proj", "model.layers.55.mlp.experts.69.gate_proj", "model.layers.55.mlp.experts.70.gate_proj", "model.layers.55.mlp.experts.71.gate_proj", "model.layers.55.mlp.experts.72.gate_proj", "model.layers.55.mlp.experts.73.gate_proj", "model.layers.55.mlp.experts.74.gate_proj", "model.layers.55.mlp.experts.75.gate_proj", "model.layers.55.mlp.experts.76.gate_proj", "model.layers.55.mlp.experts.77.gate_proj", "model.layers.55.mlp.experts.78.gate_proj", "model.layers.55.mlp.experts.79.gate_proj", "model.layers.55.mlp.experts.80.gate_proj", "model.layers.55.mlp.experts.81.gate_proj", "model.layers.55.mlp.experts.82.gate_proj", "model.layers.55.mlp.experts.83.gate_proj", "model.layers.55.mlp.experts.84.gate_proj", "model.layers.55.mlp.experts.85.gate_proj", "model.layers.55.mlp.experts.86.gate_proj", "model.layers.55.mlp.experts.87.gate_proj", "model.layers.55.mlp.experts.88.gate_proj", "model.layers.55.mlp.experts.89.gate_proj", "model.layers.55.mlp.experts.90.gate_proj", "model.layers.55.mlp.experts.91.gate_proj", "model.layers.55.mlp.experts.92.gate_proj", "model.layers.55.mlp.experts.93.gate_proj", "model.layers.55.mlp.experts.94.gate_proj", "model.layers.55.mlp.experts.95.gate_proj", "model.layers.55.mlp.experts.96.gate_proj", "model.layers.55.mlp.experts.97.gate_proj", "model.layers.55.mlp.experts.98.gate_proj", "model.layers.55.mlp.experts.99.gate_proj", "model.layers.55.mlp.experts.100.gate_proj", "model.layers.55.mlp.experts.101.gate_proj", "model.layers.55.mlp.experts.102.gate_proj", "model.layers.55.mlp.experts.103.gate_proj", "model.layers.55.mlp.experts.104.gate_proj", "model.layers.55.mlp.experts.105.gate_proj", "model.layers.55.mlp.experts.106.gate_proj", "model.layers.55.mlp.experts.107.gate_proj", "model.layers.55.mlp.experts.108.gate_proj", "model.layers.55.mlp.experts.109.gate_proj", "model.layers.55.mlp.experts.110.gate_proj", "model.layers.55.mlp.experts.111.gate_proj", "model.layers.55.mlp.experts.112.gate_proj", "model.layers.55.mlp.experts.113.gate_proj", "model.layers.55.mlp.experts.114.gate_proj", "model.layers.55.mlp.experts.115.gate_proj", "model.layers.55.mlp.experts.116.gate_proj", "model.layers.55.mlp.experts.117.gate_proj", "model.layers.55.mlp.experts.118.gate_proj", "model.layers.55.mlp.experts.119.gate_proj", "model.layers.55.mlp.experts.120.gate_proj", "model.layers.55.mlp.experts.121.gate_proj", "model.layers.55.mlp.experts.122.gate_proj", "model.layers.55.mlp.experts.123.gate_proj", "model.layers.55.mlp.experts.124.gate_proj", "model.layers.55.mlp.experts.125.gate_proj", "model.layers.55.mlp.experts.126.gate_proj", "model.layers.55.mlp.experts.127.gate_proj", "model.layers.55.mlp.experts.128.gate_proj", "model.layers.55.mlp.experts.129.gate_proj", "model.layers.55.mlp.experts.130.gate_proj", "model.layers.55.mlp.experts.131.gate_proj", "model.layers.55.mlp.experts.132.gate_proj", "model.layers.55.mlp.experts.133.gate_proj", "model.layers.55.mlp.experts.134.gate_proj", "model.layers.55.mlp.experts.135.gate_proj", "model.layers.55.mlp.experts.136.gate_proj", "model.layers.55.mlp.experts.137.gate_proj", "model.layers.55.mlp.experts.138.gate_proj", "model.layers.55.mlp.experts.139.gate_proj", "model.layers.55.mlp.experts.140.gate_proj", "model.layers.55.mlp.experts.141.gate_proj", "model.layers.55.mlp.experts.142.gate_proj", "model.layers.55.mlp.experts.143.gate_proj", "model.layers.55.mlp.experts.144.gate_proj", "model.layers.55.mlp.experts.145.gate_proj", "model.layers.55.mlp.experts.146.gate_proj", "model.layers.55.mlp.experts.147.gate_proj", "model.layers.55.mlp.experts.148.gate_proj", "model.layers.55.mlp.experts.149.gate_proj", "model.layers.55.mlp.experts.150.gate_proj", "model.layers.55.mlp.experts.151.gate_proj", "model.layers.55.mlp.experts.152.gate_proj", "model.layers.55.mlp.experts.153.gate_proj", "model.layers.55.mlp.experts.154.gate_proj", "model.layers.55.mlp.experts.155.gate_proj", "model.layers.55.mlp.experts.156.gate_proj", "model.layers.55.mlp.experts.157.gate_proj", "model.layers.55.mlp.experts.158.gate_proj", "model.layers.55.mlp.experts.159.gate_proj", "model.layers.55.mlp.experts.0.up_proj", "model.layers.55.mlp.experts.1.up_proj", "model.layers.55.mlp.experts.2.up_proj", "model.layers.55.mlp.experts.3.up_proj", "model.layers.55.mlp.experts.4.up_proj", "model.layers.55.mlp.experts.5.up_proj", "model.layers.55.mlp.experts.6.up_proj", "model.layers.55.mlp.experts.7.up_proj", "model.layers.55.mlp.experts.8.up_proj", "model.layers.55.mlp.experts.9.up_proj", "model.layers.55.mlp.experts.10.up_proj", "model.layers.55.mlp.experts.11.up_proj", "model.layers.55.mlp.experts.12.up_proj", "model.layers.55.mlp.experts.13.up_proj", "model.layers.55.mlp.experts.14.up_proj", "model.layers.55.mlp.experts.15.up_proj", "model.layers.55.mlp.experts.16.up_proj", "model.layers.55.mlp.experts.17.up_proj", "model.layers.55.mlp.experts.18.up_proj", "model.layers.55.mlp.experts.19.up_proj", "model.layers.55.mlp.experts.20.up_proj", "model.layers.55.mlp.experts.21.up_proj", "model.layers.55.mlp.experts.22.up_proj", "model.layers.55.mlp.experts.23.up_proj", "model.layers.55.mlp.experts.24.up_proj", "model.layers.55.mlp.experts.25.up_proj", "model.layers.55.mlp.experts.26.up_proj", "model.layers.55.mlp.experts.27.up_proj", "model.layers.55.mlp.experts.28.up_proj", "model.layers.55.mlp.experts.29.up_proj", "model.layers.55.mlp.experts.30.up_proj", "model.layers.55.mlp.experts.31.up_proj", "model.layers.55.mlp.experts.32.up_proj", "model.layers.55.mlp.experts.33.up_proj", "model.layers.55.mlp.experts.34.up_proj", "model.layers.55.mlp.experts.35.up_proj", "model.layers.55.mlp.experts.36.up_proj", "model.layers.55.mlp.experts.37.up_proj", "model.layers.55.mlp.experts.38.up_proj", "model.layers.55.mlp.experts.39.up_proj", "model.layers.55.mlp.experts.40.up_proj", "model.layers.55.mlp.experts.41.up_proj", "model.layers.55.mlp.experts.42.up_proj", "model.layers.55.mlp.experts.43.up_proj", "model.layers.55.mlp.experts.44.up_proj", "model.layers.55.mlp.experts.45.up_proj", "model.layers.55.mlp.experts.46.up_proj", "model.layers.55.mlp.experts.47.up_proj", "model.layers.55.mlp.experts.48.up_proj", "model.layers.55.mlp.experts.49.up_proj", "model.layers.55.mlp.experts.50.up_proj", "model.layers.55.mlp.experts.51.up_proj", "model.layers.55.mlp.experts.52.up_proj", "model.layers.55.mlp.experts.53.up_proj", "model.layers.55.mlp.experts.54.up_proj", "model.layers.55.mlp.experts.55.up_proj", "model.layers.55.mlp.experts.56.up_proj", "model.layers.55.mlp.experts.57.up_proj", "model.layers.55.mlp.experts.58.up_proj", "model.layers.55.mlp.experts.59.up_proj", "model.layers.55.mlp.experts.60.up_proj", "model.layers.55.mlp.experts.61.up_proj", "model.layers.55.mlp.experts.62.up_proj", "model.layers.55.mlp.experts.63.up_proj", "model.layers.55.mlp.experts.64.up_proj", "model.layers.55.mlp.experts.65.up_proj", "model.layers.55.mlp.experts.66.up_proj", "model.layers.55.mlp.experts.67.up_proj", "model.layers.55.mlp.experts.68.up_proj", "model.layers.55.mlp.experts.69.up_proj", "model.layers.55.mlp.experts.70.up_proj", "model.layers.55.mlp.experts.71.up_proj", "model.layers.55.mlp.experts.72.up_proj", "model.layers.55.mlp.experts.73.up_proj", "model.layers.55.mlp.experts.74.up_proj", "model.layers.55.mlp.experts.75.up_proj", "model.layers.55.mlp.experts.76.up_proj", "model.layers.55.mlp.experts.77.up_proj", "model.layers.55.mlp.experts.78.up_proj", "model.layers.55.mlp.experts.79.up_proj", "model.layers.55.mlp.experts.80.up_proj", "model.layers.55.mlp.experts.81.up_proj", "model.layers.55.mlp.experts.82.up_proj", "model.layers.55.mlp.experts.83.up_proj", "model.layers.55.mlp.experts.84.up_proj", "model.layers.55.mlp.experts.85.up_proj", "model.layers.55.mlp.experts.86.up_proj", "model.layers.55.mlp.experts.87.up_proj", "model.layers.55.mlp.experts.88.up_proj", "model.layers.55.mlp.experts.89.up_proj", "model.layers.55.mlp.experts.90.up_proj", "model.layers.55.mlp.experts.91.up_proj", "model.layers.55.mlp.experts.92.up_proj", "model.layers.55.mlp.experts.93.up_proj", "model.layers.55.mlp.experts.94.up_proj", "model.layers.55.mlp.experts.95.up_proj", "model.layers.55.mlp.experts.96.up_proj", "model.layers.55.mlp.experts.97.up_proj", "model.layers.55.mlp.experts.98.up_proj", "model.layers.55.mlp.experts.99.up_proj", "model.layers.55.mlp.experts.100.up_proj", "model.layers.55.mlp.experts.101.up_proj", "model.layers.55.mlp.experts.102.up_proj", "model.layers.55.mlp.experts.103.up_proj", "model.layers.55.mlp.experts.104.up_proj", "model.layers.55.mlp.experts.105.up_proj", "model.layers.55.mlp.experts.106.up_proj", "model.layers.55.mlp.experts.107.up_proj", "model.layers.55.mlp.experts.108.up_proj", "model.layers.55.mlp.experts.109.up_proj", "model.layers.55.mlp.experts.110.up_proj", "model.layers.55.mlp.experts.111.up_proj", "model.layers.55.mlp.experts.112.up_proj", "model.layers.55.mlp.experts.113.up_proj", "model.layers.55.mlp.experts.114.up_proj", "model.layers.55.mlp.experts.115.up_proj", "model.layers.55.mlp.experts.116.up_proj", "model.layers.55.mlp.experts.117.up_proj", "model.layers.55.mlp.experts.118.up_proj", "model.layers.55.mlp.experts.119.up_proj", "model.layers.55.mlp.experts.120.up_proj", "model.layers.55.mlp.experts.121.up_proj", "model.layers.55.mlp.experts.122.up_proj", "model.layers.55.mlp.experts.123.up_proj", "model.layers.55.mlp.experts.124.up_proj", "model.layers.55.mlp.experts.125.up_proj", "model.layers.55.mlp.experts.126.up_proj", "model.layers.55.mlp.experts.127.up_proj", "model.layers.55.mlp.experts.128.up_proj", "model.layers.55.mlp.experts.129.up_proj", "model.layers.55.mlp.experts.130.up_proj", "model.layers.55.mlp.experts.131.up_proj", "model.layers.55.mlp.experts.132.up_proj", "model.layers.55.mlp.experts.133.up_proj", "model.layers.55.mlp.experts.134.up_proj", "model.layers.55.mlp.experts.135.up_proj", "model.layers.55.mlp.experts.136.up_proj", "model.layers.55.mlp.experts.137.up_proj", "model.layers.55.mlp.experts.138.up_proj", "model.layers.55.mlp.experts.139.up_proj", "model.layers.55.mlp.experts.140.up_proj", "model.layers.55.mlp.experts.141.up_proj", "model.layers.55.mlp.experts.142.up_proj", "model.layers.55.mlp.experts.143.up_proj", "model.layers.55.mlp.experts.144.up_proj", "model.layers.55.mlp.experts.145.up_proj", "model.layers.55.mlp.experts.146.up_proj", "model.layers.55.mlp.experts.147.up_proj", "model.layers.55.mlp.experts.148.up_proj", "model.layers.55.mlp.experts.149.up_proj", "model.layers.55.mlp.experts.150.up_proj", "model.layers.55.mlp.experts.151.up_proj", "model.layers.55.mlp.experts.152.up_proj", "model.layers.55.mlp.experts.153.up_proj", "model.layers.55.mlp.experts.154.up_proj", "model.layers.55.mlp.experts.155.up_proj", "model.layers.55.mlp.experts.156.up_proj", "model.layers.55.mlp.experts.157.up_proj", "model.layers.55.mlp.experts.158.up_proj", "model.layers.55.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -3.25031578540802e-07, "dbits": 5033164800 } ] }, { "idx": 332, "layers": [ "model.layers.55.mlp.experts.0.down_proj", "model.layers.55.mlp.experts.1.down_proj", "model.layers.55.mlp.experts.2.down_proj", "model.layers.55.mlp.experts.3.down_proj", "model.layers.55.mlp.experts.4.down_proj", "model.layers.55.mlp.experts.5.down_proj", "model.layers.55.mlp.experts.6.down_proj", "model.layers.55.mlp.experts.7.down_proj", "model.layers.55.mlp.experts.8.down_proj", "model.layers.55.mlp.experts.9.down_proj", "model.layers.55.mlp.experts.10.down_proj", "model.layers.55.mlp.experts.11.down_proj", "model.layers.55.mlp.experts.12.down_proj", "model.layers.55.mlp.experts.13.down_proj", "model.layers.55.mlp.experts.14.down_proj", "model.layers.55.mlp.experts.15.down_proj", "model.layers.55.mlp.experts.16.down_proj", "model.layers.55.mlp.experts.17.down_proj", "model.layers.55.mlp.experts.18.down_proj", "model.layers.55.mlp.experts.19.down_proj", "model.layers.55.mlp.experts.20.down_proj", "model.layers.55.mlp.experts.21.down_proj", "model.layers.55.mlp.experts.22.down_proj", "model.layers.55.mlp.experts.23.down_proj", "model.layers.55.mlp.experts.24.down_proj", "model.layers.55.mlp.experts.25.down_proj", "model.layers.55.mlp.experts.26.down_proj", "model.layers.55.mlp.experts.27.down_proj", "model.layers.55.mlp.experts.28.down_proj", "model.layers.55.mlp.experts.29.down_proj", "model.layers.55.mlp.experts.30.down_proj", "model.layers.55.mlp.experts.31.down_proj", "model.layers.55.mlp.experts.32.down_proj", "model.layers.55.mlp.experts.33.down_proj", "model.layers.55.mlp.experts.34.down_proj", "model.layers.55.mlp.experts.35.down_proj", "model.layers.55.mlp.experts.36.down_proj", "model.layers.55.mlp.experts.37.down_proj", "model.layers.55.mlp.experts.38.down_proj", "model.layers.55.mlp.experts.39.down_proj", "model.layers.55.mlp.experts.40.down_proj", "model.layers.55.mlp.experts.41.down_proj", "model.layers.55.mlp.experts.42.down_proj", "model.layers.55.mlp.experts.43.down_proj", "model.layers.55.mlp.experts.44.down_proj", "model.layers.55.mlp.experts.45.down_proj", "model.layers.55.mlp.experts.46.down_proj", "model.layers.55.mlp.experts.47.down_proj", "model.layers.55.mlp.experts.48.down_proj", "model.layers.55.mlp.experts.49.down_proj", "model.layers.55.mlp.experts.50.down_proj", "model.layers.55.mlp.experts.51.down_proj", "model.layers.55.mlp.experts.52.down_proj", "model.layers.55.mlp.experts.53.down_proj", "model.layers.55.mlp.experts.54.down_proj", "model.layers.55.mlp.experts.55.down_proj", "model.layers.55.mlp.experts.56.down_proj", "model.layers.55.mlp.experts.57.down_proj", "model.layers.55.mlp.experts.58.down_proj", "model.layers.55.mlp.experts.59.down_proj", "model.layers.55.mlp.experts.60.down_proj", "model.layers.55.mlp.experts.61.down_proj", "model.layers.55.mlp.experts.62.down_proj", "model.layers.55.mlp.experts.63.down_proj", "model.layers.55.mlp.experts.64.down_proj", "model.layers.55.mlp.experts.65.down_proj", "model.layers.55.mlp.experts.66.down_proj", "model.layers.55.mlp.experts.67.down_proj", "model.layers.55.mlp.experts.68.down_proj", "model.layers.55.mlp.experts.69.down_proj", "model.layers.55.mlp.experts.70.down_proj", "model.layers.55.mlp.experts.71.down_proj", "model.layers.55.mlp.experts.72.down_proj", "model.layers.55.mlp.experts.73.down_proj", "model.layers.55.mlp.experts.74.down_proj", "model.layers.55.mlp.experts.75.down_proj", "model.layers.55.mlp.experts.76.down_proj", "model.layers.55.mlp.experts.77.down_proj", "model.layers.55.mlp.experts.78.down_proj", "model.layers.55.mlp.experts.79.down_proj", "model.layers.55.mlp.experts.80.down_proj", "model.layers.55.mlp.experts.81.down_proj", "model.layers.55.mlp.experts.82.down_proj", "model.layers.55.mlp.experts.83.down_proj", "model.layers.55.mlp.experts.84.down_proj", "model.layers.55.mlp.experts.85.down_proj", "model.layers.55.mlp.experts.86.down_proj", "model.layers.55.mlp.experts.87.down_proj", "model.layers.55.mlp.experts.88.down_proj", "model.layers.55.mlp.experts.89.down_proj", "model.layers.55.mlp.experts.90.down_proj", "model.layers.55.mlp.experts.91.down_proj", "model.layers.55.mlp.experts.92.down_proj", "model.layers.55.mlp.experts.93.down_proj", "model.layers.55.mlp.experts.94.down_proj", "model.layers.55.mlp.experts.95.down_proj", "model.layers.55.mlp.experts.96.down_proj", "model.layers.55.mlp.experts.97.down_proj", "model.layers.55.mlp.experts.98.down_proj", "model.layers.55.mlp.experts.99.down_proj", "model.layers.55.mlp.experts.100.down_proj", "model.layers.55.mlp.experts.101.down_proj", "model.layers.55.mlp.experts.102.down_proj", "model.layers.55.mlp.experts.103.down_proj", "model.layers.55.mlp.experts.104.down_proj", "model.layers.55.mlp.experts.105.down_proj", "model.layers.55.mlp.experts.106.down_proj", "model.layers.55.mlp.experts.107.down_proj", "model.layers.55.mlp.experts.108.down_proj", "model.layers.55.mlp.experts.109.down_proj", "model.layers.55.mlp.experts.110.down_proj", "model.layers.55.mlp.experts.111.down_proj", "model.layers.55.mlp.experts.112.down_proj", "model.layers.55.mlp.experts.113.down_proj", "model.layers.55.mlp.experts.114.down_proj", "model.layers.55.mlp.experts.115.down_proj", "model.layers.55.mlp.experts.116.down_proj", "model.layers.55.mlp.experts.117.down_proj", "model.layers.55.mlp.experts.118.down_proj", "model.layers.55.mlp.experts.119.down_proj", "model.layers.55.mlp.experts.120.down_proj", "model.layers.55.mlp.experts.121.down_proj", "model.layers.55.mlp.experts.122.down_proj", "model.layers.55.mlp.experts.123.down_proj", "model.layers.55.mlp.experts.124.down_proj", "model.layers.55.mlp.experts.125.down_proj", "model.layers.55.mlp.experts.126.down_proj", "model.layers.55.mlp.experts.127.down_proj", "model.layers.55.mlp.experts.128.down_proj", "model.layers.55.mlp.experts.129.down_proj", "model.layers.55.mlp.experts.130.down_proj", "model.layers.55.mlp.experts.131.down_proj", "model.layers.55.mlp.experts.132.down_proj", "model.layers.55.mlp.experts.133.down_proj", "model.layers.55.mlp.experts.134.down_proj", "model.layers.55.mlp.experts.135.down_proj", "model.layers.55.mlp.experts.136.down_proj", "model.layers.55.mlp.experts.137.down_proj", "model.layers.55.mlp.experts.138.down_proj", "model.layers.55.mlp.experts.139.down_proj", "model.layers.55.mlp.experts.140.down_proj", "model.layers.55.mlp.experts.141.down_proj", "model.layers.55.mlp.experts.142.down_proj", "model.layers.55.mlp.experts.143.down_proj", "model.layers.55.mlp.experts.144.down_proj", "model.layers.55.mlp.experts.145.down_proj", "model.layers.55.mlp.experts.146.down_proj", "model.layers.55.mlp.experts.147.down_proj", "model.layers.55.mlp.experts.148.down_proj", "model.layers.55.mlp.experts.149.down_proj", "model.layers.55.mlp.experts.150.down_proj", "model.layers.55.mlp.experts.151.down_proj", "model.layers.55.mlp.experts.152.down_proj", "model.layers.55.mlp.experts.153.down_proj", "model.layers.55.mlp.experts.154.down_proj", "model.layers.55.mlp.experts.155.down_proj", "model.layers.55.mlp.experts.156.down_proj", "model.layers.55.mlp.experts.157.down_proj", "model.layers.55.mlp.experts.158.down_proj", "model.layers.55.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.3046944988891266e-06, "dbits": 2516582400 } ] }, { "idx": 333, "layers": [ "model.layers.56.self_attn.q_proj" ], "candidates": [ { "dkld": 3.5287521313873745e-07, "dbits": 125829120 } ] }, { "idx": 334, "layers": [ "model.layers.56.self_attn.k_proj", "model.layers.56.self_attn.v_proj" ], "candidates": [ { "dkld": 1.8373626517132141e-06, "dbits": 20971520 } ] }, { "idx": 335, "layers": [ "model.layers.56.self_attn.o_proj" ], "candidates": [ { "dkld": 5.94905577600024e-07, "dbits": 125829120 } ] }, { "idx": 336, "layers": [ "model.layers.56.mlp.shared_experts.gate_proj", "model.layers.56.mlp.shared_experts.up_proj", "model.layers.56.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.5749766286462545e-06, "dbits": 47185920 } ] }, { "idx": 337, "layers": [ "model.layers.56.mlp.experts.0.gate_proj", "model.layers.56.mlp.experts.1.gate_proj", "model.layers.56.mlp.experts.2.gate_proj", "model.layers.56.mlp.experts.3.gate_proj", "model.layers.56.mlp.experts.4.gate_proj", "model.layers.56.mlp.experts.5.gate_proj", "model.layers.56.mlp.experts.6.gate_proj", "model.layers.56.mlp.experts.7.gate_proj", "model.layers.56.mlp.experts.8.gate_proj", "model.layers.56.mlp.experts.9.gate_proj", "model.layers.56.mlp.experts.10.gate_proj", "model.layers.56.mlp.experts.11.gate_proj", "model.layers.56.mlp.experts.12.gate_proj", "model.layers.56.mlp.experts.13.gate_proj", "model.layers.56.mlp.experts.14.gate_proj", "model.layers.56.mlp.experts.15.gate_proj", "model.layers.56.mlp.experts.16.gate_proj", "model.layers.56.mlp.experts.17.gate_proj", "model.layers.56.mlp.experts.18.gate_proj", "model.layers.56.mlp.experts.19.gate_proj", "model.layers.56.mlp.experts.20.gate_proj", "model.layers.56.mlp.experts.21.gate_proj", "model.layers.56.mlp.experts.22.gate_proj", "model.layers.56.mlp.experts.23.gate_proj", "model.layers.56.mlp.experts.24.gate_proj", "model.layers.56.mlp.experts.25.gate_proj", "model.layers.56.mlp.experts.26.gate_proj", "model.layers.56.mlp.experts.27.gate_proj", "model.layers.56.mlp.experts.28.gate_proj", "model.layers.56.mlp.experts.29.gate_proj", "model.layers.56.mlp.experts.30.gate_proj", "model.layers.56.mlp.experts.31.gate_proj", "model.layers.56.mlp.experts.32.gate_proj", "model.layers.56.mlp.experts.33.gate_proj", "model.layers.56.mlp.experts.34.gate_proj", "model.layers.56.mlp.experts.35.gate_proj", "model.layers.56.mlp.experts.36.gate_proj", "model.layers.56.mlp.experts.37.gate_proj", "model.layers.56.mlp.experts.38.gate_proj", "model.layers.56.mlp.experts.39.gate_proj", "model.layers.56.mlp.experts.40.gate_proj", "model.layers.56.mlp.experts.41.gate_proj", "model.layers.56.mlp.experts.42.gate_proj", "model.layers.56.mlp.experts.43.gate_proj", "model.layers.56.mlp.experts.44.gate_proj", "model.layers.56.mlp.experts.45.gate_proj", "model.layers.56.mlp.experts.46.gate_proj", "model.layers.56.mlp.experts.47.gate_proj", "model.layers.56.mlp.experts.48.gate_proj", "model.layers.56.mlp.experts.49.gate_proj", "model.layers.56.mlp.experts.50.gate_proj", "model.layers.56.mlp.experts.51.gate_proj", "model.layers.56.mlp.experts.52.gate_proj", "model.layers.56.mlp.experts.53.gate_proj", "model.layers.56.mlp.experts.54.gate_proj", "model.layers.56.mlp.experts.55.gate_proj", "model.layers.56.mlp.experts.56.gate_proj", "model.layers.56.mlp.experts.57.gate_proj", "model.layers.56.mlp.experts.58.gate_proj", "model.layers.56.mlp.experts.59.gate_proj", "model.layers.56.mlp.experts.60.gate_proj", "model.layers.56.mlp.experts.61.gate_proj", "model.layers.56.mlp.experts.62.gate_proj", "model.layers.56.mlp.experts.63.gate_proj", "model.layers.56.mlp.experts.64.gate_proj", "model.layers.56.mlp.experts.65.gate_proj", "model.layers.56.mlp.experts.66.gate_proj", "model.layers.56.mlp.experts.67.gate_proj", "model.layers.56.mlp.experts.68.gate_proj", "model.layers.56.mlp.experts.69.gate_proj", "model.layers.56.mlp.experts.70.gate_proj", "model.layers.56.mlp.experts.71.gate_proj", "model.layers.56.mlp.experts.72.gate_proj", "model.layers.56.mlp.experts.73.gate_proj", "model.layers.56.mlp.experts.74.gate_proj", "model.layers.56.mlp.experts.75.gate_proj", "model.layers.56.mlp.experts.76.gate_proj", "model.layers.56.mlp.experts.77.gate_proj", "model.layers.56.mlp.experts.78.gate_proj", "model.layers.56.mlp.experts.79.gate_proj", "model.layers.56.mlp.experts.80.gate_proj", "model.layers.56.mlp.experts.81.gate_proj", "model.layers.56.mlp.experts.82.gate_proj", "model.layers.56.mlp.experts.83.gate_proj", "model.layers.56.mlp.experts.84.gate_proj", "model.layers.56.mlp.experts.85.gate_proj", "model.layers.56.mlp.experts.86.gate_proj", "model.layers.56.mlp.experts.87.gate_proj", "model.layers.56.mlp.experts.88.gate_proj", "model.layers.56.mlp.experts.89.gate_proj", "model.layers.56.mlp.experts.90.gate_proj", "model.layers.56.mlp.experts.91.gate_proj", "model.layers.56.mlp.experts.92.gate_proj", "model.layers.56.mlp.experts.93.gate_proj", "model.layers.56.mlp.experts.94.gate_proj", "model.layers.56.mlp.experts.95.gate_proj", "model.layers.56.mlp.experts.96.gate_proj", "model.layers.56.mlp.experts.97.gate_proj", "model.layers.56.mlp.experts.98.gate_proj", "model.layers.56.mlp.experts.99.gate_proj", "model.layers.56.mlp.experts.100.gate_proj", "model.layers.56.mlp.experts.101.gate_proj", "model.layers.56.mlp.experts.102.gate_proj", "model.layers.56.mlp.experts.103.gate_proj", "model.layers.56.mlp.experts.104.gate_proj", "model.layers.56.mlp.experts.105.gate_proj", "model.layers.56.mlp.experts.106.gate_proj", "model.layers.56.mlp.experts.107.gate_proj", "model.layers.56.mlp.experts.108.gate_proj", "model.layers.56.mlp.experts.109.gate_proj", "model.layers.56.mlp.experts.110.gate_proj", "model.layers.56.mlp.experts.111.gate_proj", "model.layers.56.mlp.experts.112.gate_proj", "model.layers.56.mlp.experts.113.gate_proj", "model.layers.56.mlp.experts.114.gate_proj", "model.layers.56.mlp.experts.115.gate_proj", "model.layers.56.mlp.experts.116.gate_proj", "model.layers.56.mlp.experts.117.gate_proj", "model.layers.56.mlp.experts.118.gate_proj", "model.layers.56.mlp.experts.119.gate_proj", "model.layers.56.mlp.experts.120.gate_proj", "model.layers.56.mlp.experts.121.gate_proj", "model.layers.56.mlp.experts.122.gate_proj", "model.layers.56.mlp.experts.123.gate_proj", "model.layers.56.mlp.experts.124.gate_proj", "model.layers.56.mlp.experts.125.gate_proj", "model.layers.56.mlp.experts.126.gate_proj", "model.layers.56.mlp.experts.127.gate_proj", "model.layers.56.mlp.experts.128.gate_proj", "model.layers.56.mlp.experts.129.gate_proj", "model.layers.56.mlp.experts.130.gate_proj", "model.layers.56.mlp.experts.131.gate_proj", "model.layers.56.mlp.experts.132.gate_proj", "model.layers.56.mlp.experts.133.gate_proj", "model.layers.56.mlp.experts.134.gate_proj", "model.layers.56.mlp.experts.135.gate_proj", "model.layers.56.mlp.experts.136.gate_proj", "model.layers.56.mlp.experts.137.gate_proj", "model.layers.56.mlp.experts.138.gate_proj", "model.layers.56.mlp.experts.139.gate_proj", "model.layers.56.mlp.experts.140.gate_proj", "model.layers.56.mlp.experts.141.gate_proj", "model.layers.56.mlp.experts.142.gate_proj", "model.layers.56.mlp.experts.143.gate_proj", "model.layers.56.mlp.experts.144.gate_proj", "model.layers.56.mlp.experts.145.gate_proj", "model.layers.56.mlp.experts.146.gate_proj", "model.layers.56.mlp.experts.147.gate_proj", "model.layers.56.mlp.experts.148.gate_proj", "model.layers.56.mlp.experts.149.gate_proj", "model.layers.56.mlp.experts.150.gate_proj", "model.layers.56.mlp.experts.151.gate_proj", "model.layers.56.mlp.experts.152.gate_proj", "model.layers.56.mlp.experts.153.gate_proj", "model.layers.56.mlp.experts.154.gate_proj", "model.layers.56.mlp.experts.155.gate_proj", "model.layers.56.mlp.experts.156.gate_proj", "model.layers.56.mlp.experts.157.gate_proj", "model.layers.56.mlp.experts.158.gate_proj", "model.layers.56.mlp.experts.159.gate_proj", "model.layers.56.mlp.experts.0.up_proj", "model.layers.56.mlp.experts.1.up_proj", "model.layers.56.mlp.experts.2.up_proj", "model.layers.56.mlp.experts.3.up_proj", "model.layers.56.mlp.experts.4.up_proj", "model.layers.56.mlp.experts.5.up_proj", "model.layers.56.mlp.experts.6.up_proj", "model.layers.56.mlp.experts.7.up_proj", "model.layers.56.mlp.experts.8.up_proj", "model.layers.56.mlp.experts.9.up_proj", "model.layers.56.mlp.experts.10.up_proj", "model.layers.56.mlp.experts.11.up_proj", "model.layers.56.mlp.experts.12.up_proj", "model.layers.56.mlp.experts.13.up_proj", "model.layers.56.mlp.experts.14.up_proj", "model.layers.56.mlp.experts.15.up_proj", "model.layers.56.mlp.experts.16.up_proj", "model.layers.56.mlp.experts.17.up_proj", "model.layers.56.mlp.experts.18.up_proj", "model.layers.56.mlp.experts.19.up_proj", "model.layers.56.mlp.experts.20.up_proj", "model.layers.56.mlp.experts.21.up_proj", "model.layers.56.mlp.experts.22.up_proj", "model.layers.56.mlp.experts.23.up_proj", "model.layers.56.mlp.experts.24.up_proj", "model.layers.56.mlp.experts.25.up_proj", "model.layers.56.mlp.experts.26.up_proj", "model.layers.56.mlp.experts.27.up_proj", "model.layers.56.mlp.experts.28.up_proj", "model.layers.56.mlp.experts.29.up_proj", "model.layers.56.mlp.experts.30.up_proj", "model.layers.56.mlp.experts.31.up_proj", "model.layers.56.mlp.experts.32.up_proj", "model.layers.56.mlp.experts.33.up_proj", "model.layers.56.mlp.experts.34.up_proj", "model.layers.56.mlp.experts.35.up_proj", "model.layers.56.mlp.experts.36.up_proj", "model.layers.56.mlp.experts.37.up_proj", "model.layers.56.mlp.experts.38.up_proj", "model.layers.56.mlp.experts.39.up_proj", "model.layers.56.mlp.experts.40.up_proj", "model.layers.56.mlp.experts.41.up_proj", "model.layers.56.mlp.experts.42.up_proj", "model.layers.56.mlp.experts.43.up_proj", "model.layers.56.mlp.experts.44.up_proj", "model.layers.56.mlp.experts.45.up_proj", "model.layers.56.mlp.experts.46.up_proj", "model.layers.56.mlp.experts.47.up_proj", "model.layers.56.mlp.experts.48.up_proj", "model.layers.56.mlp.experts.49.up_proj", "model.layers.56.mlp.experts.50.up_proj", "model.layers.56.mlp.experts.51.up_proj", "model.layers.56.mlp.experts.52.up_proj", "model.layers.56.mlp.experts.53.up_proj", "model.layers.56.mlp.experts.54.up_proj", "model.layers.56.mlp.experts.55.up_proj", "model.layers.56.mlp.experts.56.up_proj", "model.layers.56.mlp.experts.57.up_proj", "model.layers.56.mlp.experts.58.up_proj", "model.layers.56.mlp.experts.59.up_proj", "model.layers.56.mlp.experts.60.up_proj", "model.layers.56.mlp.experts.61.up_proj", "model.layers.56.mlp.experts.62.up_proj", "model.layers.56.mlp.experts.63.up_proj", "model.layers.56.mlp.experts.64.up_proj", "model.layers.56.mlp.experts.65.up_proj", "model.layers.56.mlp.experts.66.up_proj", "model.layers.56.mlp.experts.67.up_proj", "model.layers.56.mlp.experts.68.up_proj", "model.layers.56.mlp.experts.69.up_proj", "model.layers.56.mlp.experts.70.up_proj", "model.layers.56.mlp.experts.71.up_proj", "model.layers.56.mlp.experts.72.up_proj", "model.layers.56.mlp.experts.73.up_proj", "model.layers.56.mlp.experts.74.up_proj", "model.layers.56.mlp.experts.75.up_proj", "model.layers.56.mlp.experts.76.up_proj", "model.layers.56.mlp.experts.77.up_proj", "model.layers.56.mlp.experts.78.up_proj", "model.layers.56.mlp.experts.79.up_proj", "model.layers.56.mlp.experts.80.up_proj", "model.layers.56.mlp.experts.81.up_proj", "model.layers.56.mlp.experts.82.up_proj", "model.layers.56.mlp.experts.83.up_proj", "model.layers.56.mlp.experts.84.up_proj", "model.layers.56.mlp.experts.85.up_proj", "model.layers.56.mlp.experts.86.up_proj", "model.layers.56.mlp.experts.87.up_proj", "model.layers.56.mlp.experts.88.up_proj", "model.layers.56.mlp.experts.89.up_proj", "model.layers.56.mlp.experts.90.up_proj", "model.layers.56.mlp.experts.91.up_proj", "model.layers.56.mlp.experts.92.up_proj", "model.layers.56.mlp.experts.93.up_proj", "model.layers.56.mlp.experts.94.up_proj", "model.layers.56.mlp.experts.95.up_proj", "model.layers.56.mlp.experts.96.up_proj", "model.layers.56.mlp.experts.97.up_proj", "model.layers.56.mlp.experts.98.up_proj", "model.layers.56.mlp.experts.99.up_proj", "model.layers.56.mlp.experts.100.up_proj", "model.layers.56.mlp.experts.101.up_proj", "model.layers.56.mlp.experts.102.up_proj", "model.layers.56.mlp.experts.103.up_proj", "model.layers.56.mlp.experts.104.up_proj", "model.layers.56.mlp.experts.105.up_proj", "model.layers.56.mlp.experts.106.up_proj", "model.layers.56.mlp.experts.107.up_proj", "model.layers.56.mlp.experts.108.up_proj", "model.layers.56.mlp.experts.109.up_proj", "model.layers.56.mlp.experts.110.up_proj", "model.layers.56.mlp.experts.111.up_proj", "model.layers.56.mlp.experts.112.up_proj", "model.layers.56.mlp.experts.113.up_proj", "model.layers.56.mlp.experts.114.up_proj", "model.layers.56.mlp.experts.115.up_proj", "model.layers.56.mlp.experts.116.up_proj", "model.layers.56.mlp.experts.117.up_proj", "model.layers.56.mlp.experts.118.up_proj", "model.layers.56.mlp.experts.119.up_proj", "model.layers.56.mlp.experts.120.up_proj", "model.layers.56.mlp.experts.121.up_proj", "model.layers.56.mlp.experts.122.up_proj", "model.layers.56.mlp.experts.123.up_proj", "model.layers.56.mlp.experts.124.up_proj", "model.layers.56.mlp.experts.125.up_proj", "model.layers.56.mlp.experts.126.up_proj", "model.layers.56.mlp.experts.127.up_proj", "model.layers.56.mlp.experts.128.up_proj", "model.layers.56.mlp.experts.129.up_proj", "model.layers.56.mlp.experts.130.up_proj", "model.layers.56.mlp.experts.131.up_proj", "model.layers.56.mlp.experts.132.up_proj", "model.layers.56.mlp.experts.133.up_proj", "model.layers.56.mlp.experts.134.up_proj", "model.layers.56.mlp.experts.135.up_proj", "model.layers.56.mlp.experts.136.up_proj", "model.layers.56.mlp.experts.137.up_proj", "model.layers.56.mlp.experts.138.up_proj", "model.layers.56.mlp.experts.139.up_proj", "model.layers.56.mlp.experts.140.up_proj", "model.layers.56.mlp.experts.141.up_proj", "model.layers.56.mlp.experts.142.up_proj", "model.layers.56.mlp.experts.143.up_proj", "model.layers.56.mlp.experts.144.up_proj", "model.layers.56.mlp.experts.145.up_proj", "model.layers.56.mlp.experts.146.up_proj", "model.layers.56.mlp.experts.147.up_proj", "model.layers.56.mlp.experts.148.up_proj", "model.layers.56.mlp.experts.149.up_proj", "model.layers.56.mlp.experts.150.up_proj", "model.layers.56.mlp.experts.151.up_proj", "model.layers.56.mlp.experts.152.up_proj", "model.layers.56.mlp.experts.153.up_proj", "model.layers.56.mlp.experts.154.up_proj", "model.layers.56.mlp.experts.155.up_proj", "model.layers.56.mlp.experts.156.up_proj", "model.layers.56.mlp.experts.157.up_proj", "model.layers.56.mlp.experts.158.up_proj", "model.layers.56.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 7.3091010563074675e-06, "dbits": 5033164800 } ] }, { "idx": 338, "layers": [ "model.layers.56.mlp.experts.0.down_proj", "model.layers.56.mlp.experts.1.down_proj", "model.layers.56.mlp.experts.2.down_proj", "model.layers.56.mlp.experts.3.down_proj", "model.layers.56.mlp.experts.4.down_proj", "model.layers.56.mlp.experts.5.down_proj", "model.layers.56.mlp.experts.6.down_proj", "model.layers.56.mlp.experts.7.down_proj", "model.layers.56.mlp.experts.8.down_proj", "model.layers.56.mlp.experts.9.down_proj", "model.layers.56.mlp.experts.10.down_proj", "model.layers.56.mlp.experts.11.down_proj", "model.layers.56.mlp.experts.12.down_proj", "model.layers.56.mlp.experts.13.down_proj", "model.layers.56.mlp.experts.14.down_proj", "model.layers.56.mlp.experts.15.down_proj", "model.layers.56.mlp.experts.16.down_proj", "model.layers.56.mlp.experts.17.down_proj", "model.layers.56.mlp.experts.18.down_proj", "model.layers.56.mlp.experts.19.down_proj", "model.layers.56.mlp.experts.20.down_proj", "model.layers.56.mlp.experts.21.down_proj", "model.layers.56.mlp.experts.22.down_proj", "model.layers.56.mlp.experts.23.down_proj", "model.layers.56.mlp.experts.24.down_proj", "model.layers.56.mlp.experts.25.down_proj", "model.layers.56.mlp.experts.26.down_proj", "model.layers.56.mlp.experts.27.down_proj", "model.layers.56.mlp.experts.28.down_proj", "model.layers.56.mlp.experts.29.down_proj", "model.layers.56.mlp.experts.30.down_proj", "model.layers.56.mlp.experts.31.down_proj", "model.layers.56.mlp.experts.32.down_proj", "model.layers.56.mlp.experts.33.down_proj", "model.layers.56.mlp.experts.34.down_proj", "model.layers.56.mlp.experts.35.down_proj", "model.layers.56.mlp.experts.36.down_proj", "model.layers.56.mlp.experts.37.down_proj", "model.layers.56.mlp.experts.38.down_proj", "model.layers.56.mlp.experts.39.down_proj", "model.layers.56.mlp.experts.40.down_proj", "model.layers.56.mlp.experts.41.down_proj", "model.layers.56.mlp.experts.42.down_proj", "model.layers.56.mlp.experts.43.down_proj", "model.layers.56.mlp.experts.44.down_proj", "model.layers.56.mlp.experts.45.down_proj", "model.layers.56.mlp.experts.46.down_proj", "model.layers.56.mlp.experts.47.down_proj", "model.layers.56.mlp.experts.48.down_proj", "model.layers.56.mlp.experts.49.down_proj", "model.layers.56.mlp.experts.50.down_proj", "model.layers.56.mlp.experts.51.down_proj", "model.layers.56.mlp.experts.52.down_proj", "model.layers.56.mlp.experts.53.down_proj", "model.layers.56.mlp.experts.54.down_proj", "model.layers.56.mlp.experts.55.down_proj", "model.layers.56.mlp.experts.56.down_proj", "model.layers.56.mlp.experts.57.down_proj", "model.layers.56.mlp.experts.58.down_proj", "model.layers.56.mlp.experts.59.down_proj", "model.layers.56.mlp.experts.60.down_proj", "model.layers.56.mlp.experts.61.down_proj", "model.layers.56.mlp.experts.62.down_proj", "model.layers.56.mlp.experts.63.down_proj", "model.layers.56.mlp.experts.64.down_proj", "model.layers.56.mlp.experts.65.down_proj", "model.layers.56.mlp.experts.66.down_proj", "model.layers.56.mlp.experts.67.down_proj", "model.layers.56.mlp.experts.68.down_proj", "model.layers.56.mlp.experts.69.down_proj", "model.layers.56.mlp.experts.70.down_proj", "model.layers.56.mlp.experts.71.down_proj", "model.layers.56.mlp.experts.72.down_proj", "model.layers.56.mlp.experts.73.down_proj", "model.layers.56.mlp.experts.74.down_proj", "model.layers.56.mlp.experts.75.down_proj", "model.layers.56.mlp.experts.76.down_proj", "model.layers.56.mlp.experts.77.down_proj", "model.layers.56.mlp.experts.78.down_proj", "model.layers.56.mlp.experts.79.down_proj", "model.layers.56.mlp.experts.80.down_proj", "model.layers.56.mlp.experts.81.down_proj", "model.layers.56.mlp.experts.82.down_proj", "model.layers.56.mlp.experts.83.down_proj", "model.layers.56.mlp.experts.84.down_proj", "model.layers.56.mlp.experts.85.down_proj", "model.layers.56.mlp.experts.86.down_proj", "model.layers.56.mlp.experts.87.down_proj", "model.layers.56.mlp.experts.88.down_proj", "model.layers.56.mlp.experts.89.down_proj", "model.layers.56.mlp.experts.90.down_proj", "model.layers.56.mlp.experts.91.down_proj", "model.layers.56.mlp.experts.92.down_proj", "model.layers.56.mlp.experts.93.down_proj", "model.layers.56.mlp.experts.94.down_proj", "model.layers.56.mlp.experts.95.down_proj", "model.layers.56.mlp.experts.96.down_proj", "model.layers.56.mlp.experts.97.down_proj", "model.layers.56.mlp.experts.98.down_proj", "model.layers.56.mlp.experts.99.down_proj", "model.layers.56.mlp.experts.100.down_proj", "model.layers.56.mlp.experts.101.down_proj", "model.layers.56.mlp.experts.102.down_proj", "model.layers.56.mlp.experts.103.down_proj", "model.layers.56.mlp.experts.104.down_proj", "model.layers.56.mlp.experts.105.down_proj", "model.layers.56.mlp.experts.106.down_proj", "model.layers.56.mlp.experts.107.down_proj", "model.layers.56.mlp.experts.108.down_proj", "model.layers.56.mlp.experts.109.down_proj", "model.layers.56.mlp.experts.110.down_proj", "model.layers.56.mlp.experts.111.down_proj", "model.layers.56.mlp.experts.112.down_proj", "model.layers.56.mlp.experts.113.down_proj", "model.layers.56.mlp.experts.114.down_proj", "model.layers.56.mlp.experts.115.down_proj", "model.layers.56.mlp.experts.116.down_proj", "model.layers.56.mlp.experts.117.down_proj", "model.layers.56.mlp.experts.118.down_proj", "model.layers.56.mlp.experts.119.down_proj", "model.layers.56.mlp.experts.120.down_proj", "model.layers.56.mlp.experts.121.down_proj", "model.layers.56.mlp.experts.122.down_proj", "model.layers.56.mlp.experts.123.down_proj", "model.layers.56.mlp.experts.124.down_proj", "model.layers.56.mlp.experts.125.down_proj", "model.layers.56.mlp.experts.126.down_proj", "model.layers.56.mlp.experts.127.down_proj", "model.layers.56.mlp.experts.128.down_proj", "model.layers.56.mlp.experts.129.down_proj", "model.layers.56.mlp.experts.130.down_proj", "model.layers.56.mlp.experts.131.down_proj", "model.layers.56.mlp.experts.132.down_proj", "model.layers.56.mlp.experts.133.down_proj", "model.layers.56.mlp.experts.134.down_proj", "model.layers.56.mlp.experts.135.down_proj", "model.layers.56.mlp.experts.136.down_proj", "model.layers.56.mlp.experts.137.down_proj", "model.layers.56.mlp.experts.138.down_proj", "model.layers.56.mlp.experts.139.down_proj", "model.layers.56.mlp.experts.140.down_proj", "model.layers.56.mlp.experts.141.down_proj", "model.layers.56.mlp.experts.142.down_proj", "model.layers.56.mlp.experts.143.down_proj", "model.layers.56.mlp.experts.144.down_proj", "model.layers.56.mlp.experts.145.down_proj", "model.layers.56.mlp.experts.146.down_proj", "model.layers.56.mlp.experts.147.down_proj", "model.layers.56.mlp.experts.148.down_proj", "model.layers.56.mlp.experts.149.down_proj", "model.layers.56.mlp.experts.150.down_proj", "model.layers.56.mlp.experts.151.down_proj", "model.layers.56.mlp.experts.152.down_proj", "model.layers.56.mlp.experts.153.down_proj", "model.layers.56.mlp.experts.154.down_proj", "model.layers.56.mlp.experts.155.down_proj", "model.layers.56.mlp.experts.156.down_proj", "model.layers.56.mlp.experts.157.down_proj", "model.layers.56.mlp.experts.158.down_proj", "model.layers.56.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.0532839698717215e-06, "dbits": 2516582400 } ] }, { "idx": 339, "layers": [ "model.layers.57.self_attn.q_proj" ], "candidates": [ { "dkld": 3.015372203662948e-06, "dbits": 125829120 } ] }, { "idx": 340, "layers": [ "model.layers.57.self_attn.k_proj", "model.layers.57.self_attn.v_proj" ], "candidates": [ { "dkld": 9.602459613234022e-07, "dbits": 20971520 } ] }, { "idx": 341, "layers": [ "model.layers.57.self_attn.o_proj" ], "candidates": [ { "dkld": 1.5191180864349008e-06, "dbits": 125829120 } ] }, { "idx": 342, "layers": [ "model.layers.57.mlp.shared_experts.gate_proj", "model.layers.57.mlp.shared_experts.up_proj", "model.layers.57.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.021888551302276e-06, "dbits": 47185920 } ] }, { "idx": 343, "layers": [ "model.layers.57.mlp.experts.0.gate_proj", "model.layers.57.mlp.experts.1.gate_proj", "model.layers.57.mlp.experts.2.gate_proj", "model.layers.57.mlp.experts.3.gate_proj", "model.layers.57.mlp.experts.4.gate_proj", "model.layers.57.mlp.experts.5.gate_proj", "model.layers.57.mlp.experts.6.gate_proj", "model.layers.57.mlp.experts.7.gate_proj", "model.layers.57.mlp.experts.8.gate_proj", "model.layers.57.mlp.experts.9.gate_proj", "model.layers.57.mlp.experts.10.gate_proj", "model.layers.57.mlp.experts.11.gate_proj", "model.layers.57.mlp.experts.12.gate_proj", "model.layers.57.mlp.experts.13.gate_proj", "model.layers.57.mlp.experts.14.gate_proj", "model.layers.57.mlp.experts.15.gate_proj", "model.layers.57.mlp.experts.16.gate_proj", "model.layers.57.mlp.experts.17.gate_proj", "model.layers.57.mlp.experts.18.gate_proj", "model.layers.57.mlp.experts.19.gate_proj", "model.layers.57.mlp.experts.20.gate_proj", "model.layers.57.mlp.experts.21.gate_proj", "model.layers.57.mlp.experts.22.gate_proj", "model.layers.57.mlp.experts.23.gate_proj", "model.layers.57.mlp.experts.24.gate_proj", "model.layers.57.mlp.experts.25.gate_proj", "model.layers.57.mlp.experts.26.gate_proj", "model.layers.57.mlp.experts.27.gate_proj", "model.layers.57.mlp.experts.28.gate_proj", "model.layers.57.mlp.experts.29.gate_proj", "model.layers.57.mlp.experts.30.gate_proj", "model.layers.57.mlp.experts.31.gate_proj", "model.layers.57.mlp.experts.32.gate_proj", "model.layers.57.mlp.experts.33.gate_proj", "model.layers.57.mlp.experts.34.gate_proj", "model.layers.57.mlp.experts.35.gate_proj", "model.layers.57.mlp.experts.36.gate_proj", "model.layers.57.mlp.experts.37.gate_proj", "model.layers.57.mlp.experts.38.gate_proj", "model.layers.57.mlp.experts.39.gate_proj", "model.layers.57.mlp.experts.40.gate_proj", "model.layers.57.mlp.experts.41.gate_proj", "model.layers.57.mlp.experts.42.gate_proj", "model.layers.57.mlp.experts.43.gate_proj", "model.layers.57.mlp.experts.44.gate_proj", "model.layers.57.mlp.experts.45.gate_proj", "model.layers.57.mlp.experts.46.gate_proj", "model.layers.57.mlp.experts.47.gate_proj", "model.layers.57.mlp.experts.48.gate_proj", "model.layers.57.mlp.experts.49.gate_proj", "model.layers.57.mlp.experts.50.gate_proj", "model.layers.57.mlp.experts.51.gate_proj", "model.layers.57.mlp.experts.52.gate_proj", "model.layers.57.mlp.experts.53.gate_proj", "model.layers.57.mlp.experts.54.gate_proj", "model.layers.57.mlp.experts.55.gate_proj", "model.layers.57.mlp.experts.56.gate_proj", "model.layers.57.mlp.experts.57.gate_proj", "model.layers.57.mlp.experts.58.gate_proj", "model.layers.57.mlp.experts.59.gate_proj", "model.layers.57.mlp.experts.60.gate_proj", "model.layers.57.mlp.experts.61.gate_proj", "model.layers.57.mlp.experts.62.gate_proj", "model.layers.57.mlp.experts.63.gate_proj", "model.layers.57.mlp.experts.64.gate_proj", "model.layers.57.mlp.experts.65.gate_proj", "model.layers.57.mlp.experts.66.gate_proj", "model.layers.57.mlp.experts.67.gate_proj", "model.layers.57.mlp.experts.68.gate_proj", "model.layers.57.mlp.experts.69.gate_proj", "model.layers.57.mlp.experts.70.gate_proj", "model.layers.57.mlp.experts.71.gate_proj", "model.layers.57.mlp.experts.72.gate_proj", "model.layers.57.mlp.experts.73.gate_proj", "model.layers.57.mlp.experts.74.gate_proj", "model.layers.57.mlp.experts.75.gate_proj", "model.layers.57.mlp.experts.76.gate_proj", "model.layers.57.mlp.experts.77.gate_proj", "model.layers.57.mlp.experts.78.gate_proj", "model.layers.57.mlp.experts.79.gate_proj", "model.layers.57.mlp.experts.80.gate_proj", "model.layers.57.mlp.experts.81.gate_proj", "model.layers.57.mlp.experts.82.gate_proj", "model.layers.57.mlp.experts.83.gate_proj", "model.layers.57.mlp.experts.84.gate_proj", "model.layers.57.mlp.experts.85.gate_proj", "model.layers.57.mlp.experts.86.gate_proj", "model.layers.57.mlp.experts.87.gate_proj", "model.layers.57.mlp.experts.88.gate_proj", "model.layers.57.mlp.experts.89.gate_proj", "model.layers.57.mlp.experts.90.gate_proj", "model.layers.57.mlp.experts.91.gate_proj", "model.layers.57.mlp.experts.92.gate_proj", "model.layers.57.mlp.experts.93.gate_proj", "model.layers.57.mlp.experts.94.gate_proj", "model.layers.57.mlp.experts.95.gate_proj", "model.layers.57.mlp.experts.96.gate_proj", "model.layers.57.mlp.experts.97.gate_proj", "model.layers.57.mlp.experts.98.gate_proj", "model.layers.57.mlp.experts.99.gate_proj", "model.layers.57.mlp.experts.100.gate_proj", "model.layers.57.mlp.experts.101.gate_proj", "model.layers.57.mlp.experts.102.gate_proj", "model.layers.57.mlp.experts.103.gate_proj", "model.layers.57.mlp.experts.104.gate_proj", "model.layers.57.mlp.experts.105.gate_proj", "model.layers.57.mlp.experts.106.gate_proj", "model.layers.57.mlp.experts.107.gate_proj", "model.layers.57.mlp.experts.108.gate_proj", "model.layers.57.mlp.experts.109.gate_proj", "model.layers.57.mlp.experts.110.gate_proj", "model.layers.57.mlp.experts.111.gate_proj", "model.layers.57.mlp.experts.112.gate_proj", "model.layers.57.mlp.experts.113.gate_proj", "model.layers.57.mlp.experts.114.gate_proj", "model.layers.57.mlp.experts.115.gate_proj", "model.layers.57.mlp.experts.116.gate_proj", "model.layers.57.mlp.experts.117.gate_proj", "model.layers.57.mlp.experts.118.gate_proj", "model.layers.57.mlp.experts.119.gate_proj", "model.layers.57.mlp.experts.120.gate_proj", "model.layers.57.mlp.experts.121.gate_proj", "model.layers.57.mlp.experts.122.gate_proj", "model.layers.57.mlp.experts.123.gate_proj", "model.layers.57.mlp.experts.124.gate_proj", "model.layers.57.mlp.experts.125.gate_proj", "model.layers.57.mlp.experts.126.gate_proj", "model.layers.57.mlp.experts.127.gate_proj", "model.layers.57.mlp.experts.128.gate_proj", "model.layers.57.mlp.experts.129.gate_proj", "model.layers.57.mlp.experts.130.gate_proj", "model.layers.57.mlp.experts.131.gate_proj", "model.layers.57.mlp.experts.132.gate_proj", "model.layers.57.mlp.experts.133.gate_proj", "model.layers.57.mlp.experts.134.gate_proj", "model.layers.57.mlp.experts.135.gate_proj", "model.layers.57.mlp.experts.136.gate_proj", "model.layers.57.mlp.experts.137.gate_proj", "model.layers.57.mlp.experts.138.gate_proj", "model.layers.57.mlp.experts.139.gate_proj", "model.layers.57.mlp.experts.140.gate_proj", "model.layers.57.mlp.experts.141.gate_proj", "model.layers.57.mlp.experts.142.gate_proj", "model.layers.57.mlp.experts.143.gate_proj", "model.layers.57.mlp.experts.144.gate_proj", "model.layers.57.mlp.experts.145.gate_proj", "model.layers.57.mlp.experts.146.gate_proj", "model.layers.57.mlp.experts.147.gate_proj", "model.layers.57.mlp.experts.148.gate_proj", "model.layers.57.mlp.experts.149.gate_proj", "model.layers.57.mlp.experts.150.gate_proj", "model.layers.57.mlp.experts.151.gate_proj", "model.layers.57.mlp.experts.152.gate_proj", "model.layers.57.mlp.experts.153.gate_proj", "model.layers.57.mlp.experts.154.gate_proj", "model.layers.57.mlp.experts.155.gate_proj", "model.layers.57.mlp.experts.156.gate_proj", "model.layers.57.mlp.experts.157.gate_proj", "model.layers.57.mlp.experts.158.gate_proj", "model.layers.57.mlp.experts.159.gate_proj", "model.layers.57.mlp.experts.0.up_proj", "model.layers.57.mlp.experts.1.up_proj", "model.layers.57.mlp.experts.2.up_proj", "model.layers.57.mlp.experts.3.up_proj", "model.layers.57.mlp.experts.4.up_proj", "model.layers.57.mlp.experts.5.up_proj", "model.layers.57.mlp.experts.6.up_proj", "model.layers.57.mlp.experts.7.up_proj", "model.layers.57.mlp.experts.8.up_proj", "model.layers.57.mlp.experts.9.up_proj", "model.layers.57.mlp.experts.10.up_proj", "model.layers.57.mlp.experts.11.up_proj", "model.layers.57.mlp.experts.12.up_proj", "model.layers.57.mlp.experts.13.up_proj", "model.layers.57.mlp.experts.14.up_proj", "model.layers.57.mlp.experts.15.up_proj", "model.layers.57.mlp.experts.16.up_proj", "model.layers.57.mlp.experts.17.up_proj", "model.layers.57.mlp.experts.18.up_proj", "model.layers.57.mlp.experts.19.up_proj", "model.layers.57.mlp.experts.20.up_proj", "model.layers.57.mlp.experts.21.up_proj", "model.layers.57.mlp.experts.22.up_proj", "model.layers.57.mlp.experts.23.up_proj", "model.layers.57.mlp.experts.24.up_proj", "model.layers.57.mlp.experts.25.up_proj", "model.layers.57.mlp.experts.26.up_proj", "model.layers.57.mlp.experts.27.up_proj", "model.layers.57.mlp.experts.28.up_proj", "model.layers.57.mlp.experts.29.up_proj", "model.layers.57.mlp.experts.30.up_proj", "model.layers.57.mlp.experts.31.up_proj", "model.layers.57.mlp.experts.32.up_proj", "model.layers.57.mlp.experts.33.up_proj", "model.layers.57.mlp.experts.34.up_proj", "model.layers.57.mlp.experts.35.up_proj", "model.layers.57.mlp.experts.36.up_proj", "model.layers.57.mlp.experts.37.up_proj", "model.layers.57.mlp.experts.38.up_proj", "model.layers.57.mlp.experts.39.up_proj", "model.layers.57.mlp.experts.40.up_proj", "model.layers.57.mlp.experts.41.up_proj", "model.layers.57.mlp.experts.42.up_proj", "model.layers.57.mlp.experts.43.up_proj", "model.layers.57.mlp.experts.44.up_proj", "model.layers.57.mlp.experts.45.up_proj", "model.layers.57.mlp.experts.46.up_proj", "model.layers.57.mlp.experts.47.up_proj", "model.layers.57.mlp.experts.48.up_proj", "model.layers.57.mlp.experts.49.up_proj", "model.layers.57.mlp.experts.50.up_proj", "model.layers.57.mlp.experts.51.up_proj", "model.layers.57.mlp.experts.52.up_proj", "model.layers.57.mlp.experts.53.up_proj", "model.layers.57.mlp.experts.54.up_proj", "model.layers.57.mlp.experts.55.up_proj", "model.layers.57.mlp.experts.56.up_proj", "model.layers.57.mlp.experts.57.up_proj", "model.layers.57.mlp.experts.58.up_proj", "model.layers.57.mlp.experts.59.up_proj", "model.layers.57.mlp.experts.60.up_proj", "model.layers.57.mlp.experts.61.up_proj", "model.layers.57.mlp.experts.62.up_proj", "model.layers.57.mlp.experts.63.up_proj", "model.layers.57.mlp.experts.64.up_proj", "model.layers.57.mlp.experts.65.up_proj", "model.layers.57.mlp.experts.66.up_proj", "model.layers.57.mlp.experts.67.up_proj", "model.layers.57.mlp.experts.68.up_proj", "model.layers.57.mlp.experts.69.up_proj", "model.layers.57.mlp.experts.70.up_proj", "model.layers.57.mlp.experts.71.up_proj", "model.layers.57.mlp.experts.72.up_proj", "model.layers.57.mlp.experts.73.up_proj", "model.layers.57.mlp.experts.74.up_proj", "model.layers.57.mlp.experts.75.up_proj", "model.layers.57.mlp.experts.76.up_proj", "model.layers.57.mlp.experts.77.up_proj", "model.layers.57.mlp.experts.78.up_proj", "model.layers.57.mlp.experts.79.up_proj", "model.layers.57.mlp.experts.80.up_proj", "model.layers.57.mlp.experts.81.up_proj", "model.layers.57.mlp.experts.82.up_proj", "model.layers.57.mlp.experts.83.up_proj", "model.layers.57.mlp.experts.84.up_proj", "model.layers.57.mlp.experts.85.up_proj", "model.layers.57.mlp.experts.86.up_proj", "model.layers.57.mlp.experts.87.up_proj", "model.layers.57.mlp.experts.88.up_proj", "model.layers.57.mlp.experts.89.up_proj", "model.layers.57.mlp.experts.90.up_proj", "model.layers.57.mlp.experts.91.up_proj", "model.layers.57.mlp.experts.92.up_proj", "model.layers.57.mlp.experts.93.up_proj", "model.layers.57.mlp.experts.94.up_proj", "model.layers.57.mlp.experts.95.up_proj", "model.layers.57.mlp.experts.96.up_proj", "model.layers.57.mlp.experts.97.up_proj", "model.layers.57.mlp.experts.98.up_proj", "model.layers.57.mlp.experts.99.up_proj", "model.layers.57.mlp.experts.100.up_proj", "model.layers.57.mlp.experts.101.up_proj", "model.layers.57.mlp.experts.102.up_proj", "model.layers.57.mlp.experts.103.up_proj", "model.layers.57.mlp.experts.104.up_proj", "model.layers.57.mlp.experts.105.up_proj", "model.layers.57.mlp.experts.106.up_proj", "model.layers.57.mlp.experts.107.up_proj", "model.layers.57.mlp.experts.108.up_proj", "model.layers.57.mlp.experts.109.up_proj", "model.layers.57.mlp.experts.110.up_proj", "model.layers.57.mlp.experts.111.up_proj", "model.layers.57.mlp.experts.112.up_proj", "model.layers.57.mlp.experts.113.up_proj", "model.layers.57.mlp.experts.114.up_proj", "model.layers.57.mlp.experts.115.up_proj", "model.layers.57.mlp.experts.116.up_proj", "model.layers.57.mlp.experts.117.up_proj", "model.layers.57.mlp.experts.118.up_proj", "model.layers.57.mlp.experts.119.up_proj", "model.layers.57.mlp.experts.120.up_proj", "model.layers.57.mlp.experts.121.up_proj", "model.layers.57.mlp.experts.122.up_proj", "model.layers.57.mlp.experts.123.up_proj", "model.layers.57.mlp.experts.124.up_proj", "model.layers.57.mlp.experts.125.up_proj", "model.layers.57.mlp.experts.126.up_proj", "model.layers.57.mlp.experts.127.up_proj", "model.layers.57.mlp.experts.128.up_proj", "model.layers.57.mlp.experts.129.up_proj", "model.layers.57.mlp.experts.130.up_proj", "model.layers.57.mlp.experts.131.up_proj", "model.layers.57.mlp.experts.132.up_proj", "model.layers.57.mlp.experts.133.up_proj", "model.layers.57.mlp.experts.134.up_proj", "model.layers.57.mlp.experts.135.up_proj", "model.layers.57.mlp.experts.136.up_proj", "model.layers.57.mlp.experts.137.up_proj", "model.layers.57.mlp.experts.138.up_proj", "model.layers.57.mlp.experts.139.up_proj", "model.layers.57.mlp.experts.140.up_proj", "model.layers.57.mlp.experts.141.up_proj", "model.layers.57.mlp.experts.142.up_proj", "model.layers.57.mlp.experts.143.up_proj", "model.layers.57.mlp.experts.144.up_proj", "model.layers.57.mlp.experts.145.up_proj", "model.layers.57.mlp.experts.146.up_proj", "model.layers.57.mlp.experts.147.up_proj", "model.layers.57.mlp.experts.148.up_proj", "model.layers.57.mlp.experts.149.up_proj", "model.layers.57.mlp.experts.150.up_proj", "model.layers.57.mlp.experts.151.up_proj", "model.layers.57.mlp.experts.152.up_proj", "model.layers.57.mlp.experts.153.up_proj", "model.layers.57.mlp.experts.154.up_proj", "model.layers.57.mlp.experts.155.up_proj", "model.layers.57.mlp.experts.156.up_proj", "model.layers.57.mlp.experts.157.up_proj", "model.layers.57.mlp.experts.158.up_proj", "model.layers.57.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 6.412519724108279e-06, "dbits": 5033164800 } ] }, { "idx": 344, "layers": [ "model.layers.57.mlp.experts.0.down_proj", "model.layers.57.mlp.experts.1.down_proj", "model.layers.57.mlp.experts.2.down_proj", "model.layers.57.mlp.experts.3.down_proj", "model.layers.57.mlp.experts.4.down_proj", "model.layers.57.mlp.experts.5.down_proj", "model.layers.57.mlp.experts.6.down_proj", "model.layers.57.mlp.experts.7.down_proj", "model.layers.57.mlp.experts.8.down_proj", "model.layers.57.mlp.experts.9.down_proj", "model.layers.57.mlp.experts.10.down_proj", "model.layers.57.mlp.experts.11.down_proj", "model.layers.57.mlp.experts.12.down_proj", "model.layers.57.mlp.experts.13.down_proj", "model.layers.57.mlp.experts.14.down_proj", "model.layers.57.mlp.experts.15.down_proj", "model.layers.57.mlp.experts.16.down_proj", "model.layers.57.mlp.experts.17.down_proj", "model.layers.57.mlp.experts.18.down_proj", "model.layers.57.mlp.experts.19.down_proj", "model.layers.57.mlp.experts.20.down_proj", "model.layers.57.mlp.experts.21.down_proj", "model.layers.57.mlp.experts.22.down_proj", "model.layers.57.mlp.experts.23.down_proj", "model.layers.57.mlp.experts.24.down_proj", "model.layers.57.mlp.experts.25.down_proj", "model.layers.57.mlp.experts.26.down_proj", "model.layers.57.mlp.experts.27.down_proj", "model.layers.57.mlp.experts.28.down_proj", "model.layers.57.mlp.experts.29.down_proj", "model.layers.57.mlp.experts.30.down_proj", "model.layers.57.mlp.experts.31.down_proj", "model.layers.57.mlp.experts.32.down_proj", "model.layers.57.mlp.experts.33.down_proj", "model.layers.57.mlp.experts.34.down_proj", "model.layers.57.mlp.experts.35.down_proj", "model.layers.57.mlp.experts.36.down_proj", "model.layers.57.mlp.experts.37.down_proj", "model.layers.57.mlp.experts.38.down_proj", "model.layers.57.mlp.experts.39.down_proj", "model.layers.57.mlp.experts.40.down_proj", "model.layers.57.mlp.experts.41.down_proj", "model.layers.57.mlp.experts.42.down_proj", "model.layers.57.mlp.experts.43.down_proj", "model.layers.57.mlp.experts.44.down_proj", "model.layers.57.mlp.experts.45.down_proj", "model.layers.57.mlp.experts.46.down_proj", "model.layers.57.mlp.experts.47.down_proj", "model.layers.57.mlp.experts.48.down_proj", "model.layers.57.mlp.experts.49.down_proj", "model.layers.57.mlp.experts.50.down_proj", "model.layers.57.mlp.experts.51.down_proj", "model.layers.57.mlp.experts.52.down_proj", "model.layers.57.mlp.experts.53.down_proj", "model.layers.57.mlp.experts.54.down_proj", "model.layers.57.mlp.experts.55.down_proj", "model.layers.57.mlp.experts.56.down_proj", "model.layers.57.mlp.experts.57.down_proj", "model.layers.57.mlp.experts.58.down_proj", "model.layers.57.mlp.experts.59.down_proj", "model.layers.57.mlp.experts.60.down_proj", "model.layers.57.mlp.experts.61.down_proj", "model.layers.57.mlp.experts.62.down_proj", "model.layers.57.mlp.experts.63.down_proj", "model.layers.57.mlp.experts.64.down_proj", "model.layers.57.mlp.experts.65.down_proj", "model.layers.57.mlp.experts.66.down_proj", "model.layers.57.mlp.experts.67.down_proj", "model.layers.57.mlp.experts.68.down_proj", "model.layers.57.mlp.experts.69.down_proj", "model.layers.57.mlp.experts.70.down_proj", "model.layers.57.mlp.experts.71.down_proj", "model.layers.57.mlp.experts.72.down_proj", "model.layers.57.mlp.experts.73.down_proj", "model.layers.57.mlp.experts.74.down_proj", "model.layers.57.mlp.experts.75.down_proj", "model.layers.57.mlp.experts.76.down_proj", "model.layers.57.mlp.experts.77.down_proj", "model.layers.57.mlp.experts.78.down_proj", "model.layers.57.mlp.experts.79.down_proj", "model.layers.57.mlp.experts.80.down_proj", "model.layers.57.mlp.experts.81.down_proj", "model.layers.57.mlp.experts.82.down_proj", "model.layers.57.mlp.experts.83.down_proj", "model.layers.57.mlp.experts.84.down_proj", "model.layers.57.mlp.experts.85.down_proj", "model.layers.57.mlp.experts.86.down_proj", "model.layers.57.mlp.experts.87.down_proj", "model.layers.57.mlp.experts.88.down_proj", "model.layers.57.mlp.experts.89.down_proj", "model.layers.57.mlp.experts.90.down_proj", "model.layers.57.mlp.experts.91.down_proj", "model.layers.57.mlp.experts.92.down_proj", "model.layers.57.mlp.experts.93.down_proj", "model.layers.57.mlp.experts.94.down_proj", "model.layers.57.mlp.experts.95.down_proj", "model.layers.57.mlp.experts.96.down_proj", "model.layers.57.mlp.experts.97.down_proj", "model.layers.57.mlp.experts.98.down_proj", "model.layers.57.mlp.experts.99.down_proj", "model.layers.57.mlp.experts.100.down_proj", "model.layers.57.mlp.experts.101.down_proj", "model.layers.57.mlp.experts.102.down_proj", "model.layers.57.mlp.experts.103.down_proj", "model.layers.57.mlp.experts.104.down_proj", "model.layers.57.mlp.experts.105.down_proj", "model.layers.57.mlp.experts.106.down_proj", "model.layers.57.mlp.experts.107.down_proj", "model.layers.57.mlp.experts.108.down_proj", "model.layers.57.mlp.experts.109.down_proj", "model.layers.57.mlp.experts.110.down_proj", "model.layers.57.mlp.experts.111.down_proj", "model.layers.57.mlp.experts.112.down_proj", "model.layers.57.mlp.experts.113.down_proj", "model.layers.57.mlp.experts.114.down_proj", "model.layers.57.mlp.experts.115.down_proj", "model.layers.57.mlp.experts.116.down_proj", "model.layers.57.mlp.experts.117.down_proj", "model.layers.57.mlp.experts.118.down_proj", "model.layers.57.mlp.experts.119.down_proj", "model.layers.57.mlp.experts.120.down_proj", "model.layers.57.mlp.experts.121.down_proj", "model.layers.57.mlp.experts.122.down_proj", "model.layers.57.mlp.experts.123.down_proj", "model.layers.57.mlp.experts.124.down_proj", "model.layers.57.mlp.experts.125.down_proj", "model.layers.57.mlp.experts.126.down_proj", "model.layers.57.mlp.experts.127.down_proj", "model.layers.57.mlp.experts.128.down_proj", "model.layers.57.mlp.experts.129.down_proj", "model.layers.57.mlp.experts.130.down_proj", "model.layers.57.mlp.experts.131.down_proj", "model.layers.57.mlp.experts.132.down_proj", "model.layers.57.mlp.experts.133.down_proj", "model.layers.57.mlp.experts.134.down_proj", "model.layers.57.mlp.experts.135.down_proj", "model.layers.57.mlp.experts.136.down_proj", "model.layers.57.mlp.experts.137.down_proj", "model.layers.57.mlp.experts.138.down_proj", "model.layers.57.mlp.experts.139.down_proj", "model.layers.57.mlp.experts.140.down_proj", "model.layers.57.mlp.experts.141.down_proj", "model.layers.57.mlp.experts.142.down_proj", "model.layers.57.mlp.experts.143.down_proj", "model.layers.57.mlp.experts.144.down_proj", "model.layers.57.mlp.experts.145.down_proj", "model.layers.57.mlp.experts.146.down_proj", "model.layers.57.mlp.experts.147.down_proj", "model.layers.57.mlp.experts.148.down_proj", "model.layers.57.mlp.experts.149.down_proj", "model.layers.57.mlp.experts.150.down_proj", "model.layers.57.mlp.experts.151.down_proj", "model.layers.57.mlp.experts.152.down_proj", "model.layers.57.mlp.experts.153.down_proj", "model.layers.57.mlp.experts.154.down_proj", "model.layers.57.mlp.experts.155.down_proj", "model.layers.57.mlp.experts.156.down_proj", "model.layers.57.mlp.experts.157.down_proj", "model.layers.57.mlp.experts.158.down_proj", "model.layers.57.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.1045922292396426e-06, "dbits": 2516582400 } ] }, { "idx": 345, "layers": [ "model.layers.58.self_attn.q_proj" ], "candidates": [ { "dkld": 2.2830412490293167e-06, "dbits": 125829120 } ] }, { "idx": 346, "layers": [ "model.layers.58.self_attn.k_proj", "model.layers.58.self_attn.v_proj" ], "candidates": [ { "dkld": 6.664922693744302e-07, "dbits": 20971520 } ] }, { "idx": 347, "layers": [ "model.layers.58.self_attn.o_proj" ], "candidates": [ { "dkld": -1.428049290552692e-06, "dbits": 125829120 } ] }, { "idx": 348, "layers": [ "model.layers.58.mlp.shared_experts.gate_proj", "model.layers.58.mlp.shared_experts.up_proj", "model.layers.58.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.138820127584062e-06, "dbits": 47185920 } ] }, { "idx": 349, "layers": [ "model.layers.58.mlp.experts.0.gate_proj", "model.layers.58.mlp.experts.1.gate_proj", "model.layers.58.mlp.experts.2.gate_proj", "model.layers.58.mlp.experts.3.gate_proj", "model.layers.58.mlp.experts.4.gate_proj", "model.layers.58.mlp.experts.5.gate_proj", "model.layers.58.mlp.experts.6.gate_proj", "model.layers.58.mlp.experts.7.gate_proj", "model.layers.58.mlp.experts.8.gate_proj", "model.layers.58.mlp.experts.9.gate_proj", "model.layers.58.mlp.experts.10.gate_proj", "model.layers.58.mlp.experts.11.gate_proj", "model.layers.58.mlp.experts.12.gate_proj", "model.layers.58.mlp.experts.13.gate_proj", "model.layers.58.mlp.experts.14.gate_proj", "model.layers.58.mlp.experts.15.gate_proj", "model.layers.58.mlp.experts.16.gate_proj", "model.layers.58.mlp.experts.17.gate_proj", "model.layers.58.mlp.experts.18.gate_proj", "model.layers.58.mlp.experts.19.gate_proj", "model.layers.58.mlp.experts.20.gate_proj", "model.layers.58.mlp.experts.21.gate_proj", "model.layers.58.mlp.experts.22.gate_proj", "model.layers.58.mlp.experts.23.gate_proj", "model.layers.58.mlp.experts.24.gate_proj", "model.layers.58.mlp.experts.25.gate_proj", "model.layers.58.mlp.experts.26.gate_proj", "model.layers.58.mlp.experts.27.gate_proj", "model.layers.58.mlp.experts.28.gate_proj", "model.layers.58.mlp.experts.29.gate_proj", "model.layers.58.mlp.experts.30.gate_proj", "model.layers.58.mlp.experts.31.gate_proj", "model.layers.58.mlp.experts.32.gate_proj", "model.layers.58.mlp.experts.33.gate_proj", "model.layers.58.mlp.experts.34.gate_proj", "model.layers.58.mlp.experts.35.gate_proj", "model.layers.58.mlp.experts.36.gate_proj", "model.layers.58.mlp.experts.37.gate_proj", "model.layers.58.mlp.experts.38.gate_proj", "model.layers.58.mlp.experts.39.gate_proj", "model.layers.58.mlp.experts.40.gate_proj", "model.layers.58.mlp.experts.41.gate_proj", "model.layers.58.mlp.experts.42.gate_proj", "model.layers.58.mlp.experts.43.gate_proj", "model.layers.58.mlp.experts.44.gate_proj", "model.layers.58.mlp.experts.45.gate_proj", "model.layers.58.mlp.experts.46.gate_proj", "model.layers.58.mlp.experts.47.gate_proj", "model.layers.58.mlp.experts.48.gate_proj", "model.layers.58.mlp.experts.49.gate_proj", "model.layers.58.mlp.experts.50.gate_proj", "model.layers.58.mlp.experts.51.gate_proj", "model.layers.58.mlp.experts.52.gate_proj", "model.layers.58.mlp.experts.53.gate_proj", "model.layers.58.mlp.experts.54.gate_proj", "model.layers.58.mlp.experts.55.gate_proj", "model.layers.58.mlp.experts.56.gate_proj", "model.layers.58.mlp.experts.57.gate_proj", "model.layers.58.mlp.experts.58.gate_proj", "model.layers.58.mlp.experts.59.gate_proj", "model.layers.58.mlp.experts.60.gate_proj", "model.layers.58.mlp.experts.61.gate_proj", "model.layers.58.mlp.experts.62.gate_proj", "model.layers.58.mlp.experts.63.gate_proj", "model.layers.58.mlp.experts.64.gate_proj", "model.layers.58.mlp.experts.65.gate_proj", "model.layers.58.mlp.experts.66.gate_proj", "model.layers.58.mlp.experts.67.gate_proj", "model.layers.58.mlp.experts.68.gate_proj", "model.layers.58.mlp.experts.69.gate_proj", "model.layers.58.mlp.experts.70.gate_proj", "model.layers.58.mlp.experts.71.gate_proj", "model.layers.58.mlp.experts.72.gate_proj", "model.layers.58.mlp.experts.73.gate_proj", "model.layers.58.mlp.experts.74.gate_proj", "model.layers.58.mlp.experts.75.gate_proj", "model.layers.58.mlp.experts.76.gate_proj", "model.layers.58.mlp.experts.77.gate_proj", "model.layers.58.mlp.experts.78.gate_proj", "model.layers.58.mlp.experts.79.gate_proj", "model.layers.58.mlp.experts.80.gate_proj", "model.layers.58.mlp.experts.81.gate_proj", "model.layers.58.mlp.experts.82.gate_proj", "model.layers.58.mlp.experts.83.gate_proj", "model.layers.58.mlp.experts.84.gate_proj", "model.layers.58.mlp.experts.85.gate_proj", "model.layers.58.mlp.experts.86.gate_proj", "model.layers.58.mlp.experts.87.gate_proj", "model.layers.58.mlp.experts.88.gate_proj", "model.layers.58.mlp.experts.89.gate_proj", "model.layers.58.mlp.experts.90.gate_proj", "model.layers.58.mlp.experts.91.gate_proj", "model.layers.58.mlp.experts.92.gate_proj", "model.layers.58.mlp.experts.93.gate_proj", "model.layers.58.mlp.experts.94.gate_proj", "model.layers.58.mlp.experts.95.gate_proj", "model.layers.58.mlp.experts.96.gate_proj", "model.layers.58.mlp.experts.97.gate_proj", "model.layers.58.mlp.experts.98.gate_proj", "model.layers.58.mlp.experts.99.gate_proj", "model.layers.58.mlp.experts.100.gate_proj", "model.layers.58.mlp.experts.101.gate_proj", "model.layers.58.mlp.experts.102.gate_proj", "model.layers.58.mlp.experts.103.gate_proj", "model.layers.58.mlp.experts.104.gate_proj", "model.layers.58.mlp.experts.105.gate_proj", "model.layers.58.mlp.experts.106.gate_proj", "model.layers.58.mlp.experts.107.gate_proj", "model.layers.58.mlp.experts.108.gate_proj", "model.layers.58.mlp.experts.109.gate_proj", "model.layers.58.mlp.experts.110.gate_proj", "model.layers.58.mlp.experts.111.gate_proj", "model.layers.58.mlp.experts.112.gate_proj", "model.layers.58.mlp.experts.113.gate_proj", "model.layers.58.mlp.experts.114.gate_proj", "model.layers.58.mlp.experts.115.gate_proj", "model.layers.58.mlp.experts.116.gate_proj", "model.layers.58.mlp.experts.117.gate_proj", "model.layers.58.mlp.experts.118.gate_proj", "model.layers.58.mlp.experts.119.gate_proj", "model.layers.58.mlp.experts.120.gate_proj", "model.layers.58.mlp.experts.121.gate_proj", "model.layers.58.mlp.experts.122.gate_proj", "model.layers.58.mlp.experts.123.gate_proj", "model.layers.58.mlp.experts.124.gate_proj", "model.layers.58.mlp.experts.125.gate_proj", "model.layers.58.mlp.experts.126.gate_proj", "model.layers.58.mlp.experts.127.gate_proj", "model.layers.58.mlp.experts.128.gate_proj", "model.layers.58.mlp.experts.129.gate_proj", "model.layers.58.mlp.experts.130.gate_proj", "model.layers.58.mlp.experts.131.gate_proj", "model.layers.58.mlp.experts.132.gate_proj", "model.layers.58.mlp.experts.133.gate_proj", "model.layers.58.mlp.experts.134.gate_proj", "model.layers.58.mlp.experts.135.gate_proj", "model.layers.58.mlp.experts.136.gate_proj", "model.layers.58.mlp.experts.137.gate_proj", "model.layers.58.mlp.experts.138.gate_proj", "model.layers.58.mlp.experts.139.gate_proj", "model.layers.58.mlp.experts.140.gate_proj", "model.layers.58.mlp.experts.141.gate_proj", "model.layers.58.mlp.experts.142.gate_proj", "model.layers.58.mlp.experts.143.gate_proj", "model.layers.58.mlp.experts.144.gate_proj", "model.layers.58.mlp.experts.145.gate_proj", "model.layers.58.mlp.experts.146.gate_proj", "model.layers.58.mlp.experts.147.gate_proj", "model.layers.58.mlp.experts.148.gate_proj", "model.layers.58.mlp.experts.149.gate_proj", "model.layers.58.mlp.experts.150.gate_proj", "model.layers.58.mlp.experts.151.gate_proj", "model.layers.58.mlp.experts.152.gate_proj", "model.layers.58.mlp.experts.153.gate_proj", "model.layers.58.mlp.experts.154.gate_proj", "model.layers.58.mlp.experts.155.gate_proj", "model.layers.58.mlp.experts.156.gate_proj", "model.layers.58.mlp.experts.157.gate_proj", "model.layers.58.mlp.experts.158.gate_proj", "model.layers.58.mlp.experts.159.gate_proj", "model.layers.58.mlp.experts.0.up_proj", "model.layers.58.mlp.experts.1.up_proj", "model.layers.58.mlp.experts.2.up_proj", "model.layers.58.mlp.experts.3.up_proj", "model.layers.58.mlp.experts.4.up_proj", "model.layers.58.mlp.experts.5.up_proj", "model.layers.58.mlp.experts.6.up_proj", "model.layers.58.mlp.experts.7.up_proj", "model.layers.58.mlp.experts.8.up_proj", "model.layers.58.mlp.experts.9.up_proj", "model.layers.58.mlp.experts.10.up_proj", "model.layers.58.mlp.experts.11.up_proj", "model.layers.58.mlp.experts.12.up_proj", "model.layers.58.mlp.experts.13.up_proj", "model.layers.58.mlp.experts.14.up_proj", "model.layers.58.mlp.experts.15.up_proj", "model.layers.58.mlp.experts.16.up_proj", "model.layers.58.mlp.experts.17.up_proj", "model.layers.58.mlp.experts.18.up_proj", "model.layers.58.mlp.experts.19.up_proj", "model.layers.58.mlp.experts.20.up_proj", "model.layers.58.mlp.experts.21.up_proj", "model.layers.58.mlp.experts.22.up_proj", "model.layers.58.mlp.experts.23.up_proj", "model.layers.58.mlp.experts.24.up_proj", "model.layers.58.mlp.experts.25.up_proj", "model.layers.58.mlp.experts.26.up_proj", "model.layers.58.mlp.experts.27.up_proj", "model.layers.58.mlp.experts.28.up_proj", "model.layers.58.mlp.experts.29.up_proj", "model.layers.58.mlp.experts.30.up_proj", "model.layers.58.mlp.experts.31.up_proj", "model.layers.58.mlp.experts.32.up_proj", "model.layers.58.mlp.experts.33.up_proj", "model.layers.58.mlp.experts.34.up_proj", "model.layers.58.mlp.experts.35.up_proj", "model.layers.58.mlp.experts.36.up_proj", "model.layers.58.mlp.experts.37.up_proj", "model.layers.58.mlp.experts.38.up_proj", "model.layers.58.mlp.experts.39.up_proj", "model.layers.58.mlp.experts.40.up_proj", "model.layers.58.mlp.experts.41.up_proj", "model.layers.58.mlp.experts.42.up_proj", "model.layers.58.mlp.experts.43.up_proj", "model.layers.58.mlp.experts.44.up_proj", "model.layers.58.mlp.experts.45.up_proj", "model.layers.58.mlp.experts.46.up_proj", "model.layers.58.mlp.experts.47.up_proj", "model.layers.58.mlp.experts.48.up_proj", "model.layers.58.mlp.experts.49.up_proj", "model.layers.58.mlp.experts.50.up_proj", "model.layers.58.mlp.experts.51.up_proj", "model.layers.58.mlp.experts.52.up_proj", "model.layers.58.mlp.experts.53.up_proj", "model.layers.58.mlp.experts.54.up_proj", "model.layers.58.mlp.experts.55.up_proj", "model.layers.58.mlp.experts.56.up_proj", "model.layers.58.mlp.experts.57.up_proj", "model.layers.58.mlp.experts.58.up_proj", "model.layers.58.mlp.experts.59.up_proj", "model.layers.58.mlp.experts.60.up_proj", "model.layers.58.mlp.experts.61.up_proj", "model.layers.58.mlp.experts.62.up_proj", "model.layers.58.mlp.experts.63.up_proj", "model.layers.58.mlp.experts.64.up_proj", "model.layers.58.mlp.experts.65.up_proj", "model.layers.58.mlp.experts.66.up_proj", "model.layers.58.mlp.experts.67.up_proj", "model.layers.58.mlp.experts.68.up_proj", "model.layers.58.mlp.experts.69.up_proj", "model.layers.58.mlp.experts.70.up_proj", "model.layers.58.mlp.experts.71.up_proj", "model.layers.58.mlp.experts.72.up_proj", "model.layers.58.mlp.experts.73.up_proj", "model.layers.58.mlp.experts.74.up_proj", "model.layers.58.mlp.experts.75.up_proj", "model.layers.58.mlp.experts.76.up_proj", "model.layers.58.mlp.experts.77.up_proj", "model.layers.58.mlp.experts.78.up_proj", "model.layers.58.mlp.experts.79.up_proj", "model.layers.58.mlp.experts.80.up_proj", "model.layers.58.mlp.experts.81.up_proj", "model.layers.58.mlp.experts.82.up_proj", "model.layers.58.mlp.experts.83.up_proj", "model.layers.58.mlp.experts.84.up_proj", "model.layers.58.mlp.experts.85.up_proj", "model.layers.58.mlp.experts.86.up_proj", "model.layers.58.mlp.experts.87.up_proj", "model.layers.58.mlp.experts.88.up_proj", "model.layers.58.mlp.experts.89.up_proj", "model.layers.58.mlp.experts.90.up_proj", "model.layers.58.mlp.experts.91.up_proj", "model.layers.58.mlp.experts.92.up_proj", "model.layers.58.mlp.experts.93.up_proj", "model.layers.58.mlp.experts.94.up_proj", "model.layers.58.mlp.experts.95.up_proj", "model.layers.58.mlp.experts.96.up_proj", "model.layers.58.mlp.experts.97.up_proj", "model.layers.58.mlp.experts.98.up_proj", "model.layers.58.mlp.experts.99.up_proj", "model.layers.58.mlp.experts.100.up_proj", "model.layers.58.mlp.experts.101.up_proj", "model.layers.58.mlp.experts.102.up_proj", "model.layers.58.mlp.experts.103.up_proj", "model.layers.58.mlp.experts.104.up_proj", "model.layers.58.mlp.experts.105.up_proj", "model.layers.58.mlp.experts.106.up_proj", "model.layers.58.mlp.experts.107.up_proj", "model.layers.58.mlp.experts.108.up_proj", "model.layers.58.mlp.experts.109.up_proj", "model.layers.58.mlp.experts.110.up_proj", "model.layers.58.mlp.experts.111.up_proj", "model.layers.58.mlp.experts.112.up_proj", "model.layers.58.mlp.experts.113.up_proj", "model.layers.58.mlp.experts.114.up_proj", "model.layers.58.mlp.experts.115.up_proj", "model.layers.58.mlp.experts.116.up_proj", "model.layers.58.mlp.experts.117.up_proj", "model.layers.58.mlp.experts.118.up_proj", "model.layers.58.mlp.experts.119.up_proj", "model.layers.58.mlp.experts.120.up_proj", "model.layers.58.mlp.experts.121.up_proj", "model.layers.58.mlp.experts.122.up_proj", "model.layers.58.mlp.experts.123.up_proj", "model.layers.58.mlp.experts.124.up_proj", "model.layers.58.mlp.experts.125.up_proj", "model.layers.58.mlp.experts.126.up_proj", "model.layers.58.mlp.experts.127.up_proj", "model.layers.58.mlp.experts.128.up_proj", "model.layers.58.mlp.experts.129.up_proj", "model.layers.58.mlp.experts.130.up_proj", "model.layers.58.mlp.experts.131.up_proj", "model.layers.58.mlp.experts.132.up_proj", "model.layers.58.mlp.experts.133.up_proj", "model.layers.58.mlp.experts.134.up_proj", "model.layers.58.mlp.experts.135.up_proj", "model.layers.58.mlp.experts.136.up_proj", "model.layers.58.mlp.experts.137.up_proj", "model.layers.58.mlp.experts.138.up_proj", "model.layers.58.mlp.experts.139.up_proj", "model.layers.58.mlp.experts.140.up_proj", "model.layers.58.mlp.experts.141.up_proj", "model.layers.58.mlp.experts.142.up_proj", "model.layers.58.mlp.experts.143.up_proj", "model.layers.58.mlp.experts.144.up_proj", "model.layers.58.mlp.experts.145.up_proj", "model.layers.58.mlp.experts.146.up_proj", "model.layers.58.mlp.experts.147.up_proj", "model.layers.58.mlp.experts.148.up_proj", "model.layers.58.mlp.experts.149.up_proj", "model.layers.58.mlp.experts.150.up_proj", "model.layers.58.mlp.experts.151.up_proj", "model.layers.58.mlp.experts.152.up_proj", "model.layers.58.mlp.experts.153.up_proj", "model.layers.58.mlp.experts.154.up_proj", "model.layers.58.mlp.experts.155.up_proj", "model.layers.58.mlp.experts.156.up_proj", "model.layers.58.mlp.experts.157.up_proj", "model.layers.58.mlp.experts.158.up_proj", "model.layers.58.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.9218266718089364e-06, "dbits": 5033164800 } ] }, { "idx": 350, "layers": [ "model.layers.58.mlp.experts.0.down_proj", "model.layers.58.mlp.experts.1.down_proj", "model.layers.58.mlp.experts.2.down_proj", "model.layers.58.mlp.experts.3.down_proj", "model.layers.58.mlp.experts.4.down_proj", "model.layers.58.mlp.experts.5.down_proj", "model.layers.58.mlp.experts.6.down_proj", "model.layers.58.mlp.experts.7.down_proj", "model.layers.58.mlp.experts.8.down_proj", "model.layers.58.mlp.experts.9.down_proj", "model.layers.58.mlp.experts.10.down_proj", "model.layers.58.mlp.experts.11.down_proj", "model.layers.58.mlp.experts.12.down_proj", "model.layers.58.mlp.experts.13.down_proj", "model.layers.58.mlp.experts.14.down_proj", "model.layers.58.mlp.experts.15.down_proj", "model.layers.58.mlp.experts.16.down_proj", "model.layers.58.mlp.experts.17.down_proj", "model.layers.58.mlp.experts.18.down_proj", "model.layers.58.mlp.experts.19.down_proj", "model.layers.58.mlp.experts.20.down_proj", "model.layers.58.mlp.experts.21.down_proj", "model.layers.58.mlp.experts.22.down_proj", "model.layers.58.mlp.experts.23.down_proj", "model.layers.58.mlp.experts.24.down_proj", "model.layers.58.mlp.experts.25.down_proj", "model.layers.58.mlp.experts.26.down_proj", "model.layers.58.mlp.experts.27.down_proj", "model.layers.58.mlp.experts.28.down_proj", "model.layers.58.mlp.experts.29.down_proj", "model.layers.58.mlp.experts.30.down_proj", "model.layers.58.mlp.experts.31.down_proj", "model.layers.58.mlp.experts.32.down_proj", "model.layers.58.mlp.experts.33.down_proj", "model.layers.58.mlp.experts.34.down_proj", "model.layers.58.mlp.experts.35.down_proj", "model.layers.58.mlp.experts.36.down_proj", "model.layers.58.mlp.experts.37.down_proj", "model.layers.58.mlp.experts.38.down_proj", "model.layers.58.mlp.experts.39.down_proj", "model.layers.58.mlp.experts.40.down_proj", "model.layers.58.mlp.experts.41.down_proj", "model.layers.58.mlp.experts.42.down_proj", "model.layers.58.mlp.experts.43.down_proj", "model.layers.58.mlp.experts.44.down_proj", "model.layers.58.mlp.experts.45.down_proj", "model.layers.58.mlp.experts.46.down_proj", "model.layers.58.mlp.experts.47.down_proj", "model.layers.58.mlp.experts.48.down_proj", "model.layers.58.mlp.experts.49.down_proj", "model.layers.58.mlp.experts.50.down_proj", "model.layers.58.mlp.experts.51.down_proj", "model.layers.58.mlp.experts.52.down_proj", "model.layers.58.mlp.experts.53.down_proj", "model.layers.58.mlp.experts.54.down_proj", "model.layers.58.mlp.experts.55.down_proj", "model.layers.58.mlp.experts.56.down_proj", "model.layers.58.mlp.experts.57.down_proj", "model.layers.58.mlp.experts.58.down_proj", "model.layers.58.mlp.experts.59.down_proj", "model.layers.58.mlp.experts.60.down_proj", "model.layers.58.mlp.experts.61.down_proj", "model.layers.58.mlp.experts.62.down_proj", "model.layers.58.mlp.experts.63.down_proj", "model.layers.58.mlp.experts.64.down_proj", "model.layers.58.mlp.experts.65.down_proj", "model.layers.58.mlp.experts.66.down_proj", "model.layers.58.mlp.experts.67.down_proj", "model.layers.58.mlp.experts.68.down_proj", "model.layers.58.mlp.experts.69.down_proj", "model.layers.58.mlp.experts.70.down_proj", "model.layers.58.mlp.experts.71.down_proj", "model.layers.58.mlp.experts.72.down_proj", "model.layers.58.mlp.experts.73.down_proj", "model.layers.58.mlp.experts.74.down_proj", "model.layers.58.mlp.experts.75.down_proj", "model.layers.58.mlp.experts.76.down_proj", "model.layers.58.mlp.experts.77.down_proj", "model.layers.58.mlp.experts.78.down_proj", "model.layers.58.mlp.experts.79.down_proj", "model.layers.58.mlp.experts.80.down_proj", "model.layers.58.mlp.experts.81.down_proj", "model.layers.58.mlp.experts.82.down_proj", "model.layers.58.mlp.experts.83.down_proj", "model.layers.58.mlp.experts.84.down_proj", "model.layers.58.mlp.experts.85.down_proj", "model.layers.58.mlp.experts.86.down_proj", "model.layers.58.mlp.experts.87.down_proj", "model.layers.58.mlp.experts.88.down_proj", "model.layers.58.mlp.experts.89.down_proj", "model.layers.58.mlp.experts.90.down_proj", "model.layers.58.mlp.experts.91.down_proj", "model.layers.58.mlp.experts.92.down_proj", "model.layers.58.mlp.experts.93.down_proj", "model.layers.58.mlp.experts.94.down_proj", "model.layers.58.mlp.experts.95.down_proj", "model.layers.58.mlp.experts.96.down_proj", "model.layers.58.mlp.experts.97.down_proj", "model.layers.58.mlp.experts.98.down_proj", "model.layers.58.mlp.experts.99.down_proj", "model.layers.58.mlp.experts.100.down_proj", "model.layers.58.mlp.experts.101.down_proj", "model.layers.58.mlp.experts.102.down_proj", "model.layers.58.mlp.experts.103.down_proj", "model.layers.58.mlp.experts.104.down_proj", "model.layers.58.mlp.experts.105.down_proj", "model.layers.58.mlp.experts.106.down_proj", "model.layers.58.mlp.experts.107.down_proj", "model.layers.58.mlp.experts.108.down_proj", "model.layers.58.mlp.experts.109.down_proj", "model.layers.58.mlp.experts.110.down_proj", "model.layers.58.mlp.experts.111.down_proj", "model.layers.58.mlp.experts.112.down_proj", "model.layers.58.mlp.experts.113.down_proj", "model.layers.58.mlp.experts.114.down_proj", "model.layers.58.mlp.experts.115.down_proj", "model.layers.58.mlp.experts.116.down_proj", "model.layers.58.mlp.experts.117.down_proj", "model.layers.58.mlp.experts.118.down_proj", "model.layers.58.mlp.experts.119.down_proj", "model.layers.58.mlp.experts.120.down_proj", "model.layers.58.mlp.experts.121.down_proj", "model.layers.58.mlp.experts.122.down_proj", "model.layers.58.mlp.experts.123.down_proj", "model.layers.58.mlp.experts.124.down_proj", "model.layers.58.mlp.experts.125.down_proj", "model.layers.58.mlp.experts.126.down_proj", "model.layers.58.mlp.experts.127.down_proj", "model.layers.58.mlp.experts.128.down_proj", "model.layers.58.mlp.experts.129.down_proj", "model.layers.58.mlp.experts.130.down_proj", "model.layers.58.mlp.experts.131.down_proj", "model.layers.58.mlp.experts.132.down_proj", "model.layers.58.mlp.experts.133.down_proj", "model.layers.58.mlp.experts.134.down_proj", "model.layers.58.mlp.experts.135.down_proj", "model.layers.58.mlp.experts.136.down_proj", "model.layers.58.mlp.experts.137.down_proj", "model.layers.58.mlp.experts.138.down_proj", "model.layers.58.mlp.experts.139.down_proj", "model.layers.58.mlp.experts.140.down_proj", "model.layers.58.mlp.experts.141.down_proj", "model.layers.58.mlp.experts.142.down_proj", "model.layers.58.mlp.experts.143.down_proj", "model.layers.58.mlp.experts.144.down_proj", "model.layers.58.mlp.experts.145.down_proj", "model.layers.58.mlp.experts.146.down_proj", "model.layers.58.mlp.experts.147.down_proj", "model.layers.58.mlp.experts.148.down_proj", "model.layers.58.mlp.experts.149.down_proj", "model.layers.58.mlp.experts.150.down_proj", "model.layers.58.mlp.experts.151.down_proj", "model.layers.58.mlp.experts.152.down_proj", "model.layers.58.mlp.experts.153.down_proj", "model.layers.58.mlp.experts.154.down_proj", "model.layers.58.mlp.experts.155.down_proj", "model.layers.58.mlp.experts.156.down_proj", "model.layers.58.mlp.experts.157.down_proj", "model.layers.58.mlp.experts.158.down_proj", "model.layers.58.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.3310462236403985e-06, "dbits": 2516582400 } ] }, { "idx": 351, "layers": [ "model.layers.59.self_attn.q_proj" ], "candidates": [ { "dkld": 2.6426627300679467e-06, "dbits": 125829120 } ] }, { "idx": 352, "layers": [ "model.layers.59.self_attn.k_proj", "model.layers.59.self_attn.v_proj" ], "candidates": [ { "dkld": 2.2852473193779588e-06, "dbits": 20971520 } ] }, { "idx": 353, "layers": [ "model.layers.59.self_attn.o_proj" ], "candidates": [ { "dkld": 1.7054291674867706e-06, "dbits": 125829120 } ] }, { "idx": 354, "layers": [ "model.layers.59.mlp.shared_experts.gate_proj", "model.layers.59.mlp.shared_experts.up_proj", "model.layers.59.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.529835910536349e-06, "dbits": 47185920 } ] }, { "idx": 355, "layers": [ "model.layers.59.mlp.experts.0.gate_proj", "model.layers.59.mlp.experts.1.gate_proj", "model.layers.59.mlp.experts.2.gate_proj", "model.layers.59.mlp.experts.3.gate_proj", "model.layers.59.mlp.experts.4.gate_proj", "model.layers.59.mlp.experts.5.gate_proj", "model.layers.59.mlp.experts.6.gate_proj", "model.layers.59.mlp.experts.7.gate_proj", "model.layers.59.mlp.experts.8.gate_proj", "model.layers.59.mlp.experts.9.gate_proj", "model.layers.59.mlp.experts.10.gate_proj", "model.layers.59.mlp.experts.11.gate_proj", "model.layers.59.mlp.experts.12.gate_proj", "model.layers.59.mlp.experts.13.gate_proj", "model.layers.59.mlp.experts.14.gate_proj", "model.layers.59.mlp.experts.15.gate_proj", "model.layers.59.mlp.experts.16.gate_proj", "model.layers.59.mlp.experts.17.gate_proj", "model.layers.59.mlp.experts.18.gate_proj", "model.layers.59.mlp.experts.19.gate_proj", "model.layers.59.mlp.experts.20.gate_proj", "model.layers.59.mlp.experts.21.gate_proj", "model.layers.59.mlp.experts.22.gate_proj", "model.layers.59.mlp.experts.23.gate_proj", "model.layers.59.mlp.experts.24.gate_proj", "model.layers.59.mlp.experts.25.gate_proj", "model.layers.59.mlp.experts.26.gate_proj", "model.layers.59.mlp.experts.27.gate_proj", "model.layers.59.mlp.experts.28.gate_proj", "model.layers.59.mlp.experts.29.gate_proj", "model.layers.59.mlp.experts.30.gate_proj", "model.layers.59.mlp.experts.31.gate_proj", "model.layers.59.mlp.experts.32.gate_proj", "model.layers.59.mlp.experts.33.gate_proj", "model.layers.59.mlp.experts.34.gate_proj", "model.layers.59.mlp.experts.35.gate_proj", "model.layers.59.mlp.experts.36.gate_proj", "model.layers.59.mlp.experts.37.gate_proj", "model.layers.59.mlp.experts.38.gate_proj", "model.layers.59.mlp.experts.39.gate_proj", "model.layers.59.mlp.experts.40.gate_proj", "model.layers.59.mlp.experts.41.gate_proj", "model.layers.59.mlp.experts.42.gate_proj", "model.layers.59.mlp.experts.43.gate_proj", "model.layers.59.mlp.experts.44.gate_proj", "model.layers.59.mlp.experts.45.gate_proj", "model.layers.59.mlp.experts.46.gate_proj", "model.layers.59.mlp.experts.47.gate_proj", "model.layers.59.mlp.experts.48.gate_proj", "model.layers.59.mlp.experts.49.gate_proj", "model.layers.59.mlp.experts.50.gate_proj", "model.layers.59.mlp.experts.51.gate_proj", "model.layers.59.mlp.experts.52.gate_proj", "model.layers.59.mlp.experts.53.gate_proj", "model.layers.59.mlp.experts.54.gate_proj", "model.layers.59.mlp.experts.55.gate_proj", "model.layers.59.mlp.experts.56.gate_proj", "model.layers.59.mlp.experts.57.gate_proj", "model.layers.59.mlp.experts.58.gate_proj", "model.layers.59.mlp.experts.59.gate_proj", "model.layers.59.mlp.experts.60.gate_proj", "model.layers.59.mlp.experts.61.gate_proj", "model.layers.59.mlp.experts.62.gate_proj", "model.layers.59.mlp.experts.63.gate_proj", "model.layers.59.mlp.experts.64.gate_proj", "model.layers.59.mlp.experts.65.gate_proj", "model.layers.59.mlp.experts.66.gate_proj", "model.layers.59.mlp.experts.67.gate_proj", "model.layers.59.mlp.experts.68.gate_proj", "model.layers.59.mlp.experts.69.gate_proj", "model.layers.59.mlp.experts.70.gate_proj", "model.layers.59.mlp.experts.71.gate_proj", "model.layers.59.mlp.experts.72.gate_proj", "model.layers.59.mlp.experts.73.gate_proj", "model.layers.59.mlp.experts.74.gate_proj", "model.layers.59.mlp.experts.75.gate_proj", "model.layers.59.mlp.experts.76.gate_proj", "model.layers.59.mlp.experts.77.gate_proj", "model.layers.59.mlp.experts.78.gate_proj", "model.layers.59.mlp.experts.79.gate_proj", "model.layers.59.mlp.experts.80.gate_proj", "model.layers.59.mlp.experts.81.gate_proj", "model.layers.59.mlp.experts.82.gate_proj", "model.layers.59.mlp.experts.83.gate_proj", "model.layers.59.mlp.experts.84.gate_proj", "model.layers.59.mlp.experts.85.gate_proj", "model.layers.59.mlp.experts.86.gate_proj", "model.layers.59.mlp.experts.87.gate_proj", "model.layers.59.mlp.experts.88.gate_proj", "model.layers.59.mlp.experts.89.gate_proj", "model.layers.59.mlp.experts.90.gate_proj", "model.layers.59.mlp.experts.91.gate_proj", "model.layers.59.mlp.experts.92.gate_proj", "model.layers.59.mlp.experts.93.gate_proj", "model.layers.59.mlp.experts.94.gate_proj", "model.layers.59.mlp.experts.95.gate_proj", "model.layers.59.mlp.experts.96.gate_proj", "model.layers.59.mlp.experts.97.gate_proj", "model.layers.59.mlp.experts.98.gate_proj", "model.layers.59.mlp.experts.99.gate_proj", "model.layers.59.mlp.experts.100.gate_proj", "model.layers.59.mlp.experts.101.gate_proj", "model.layers.59.mlp.experts.102.gate_proj", "model.layers.59.mlp.experts.103.gate_proj", "model.layers.59.mlp.experts.104.gate_proj", "model.layers.59.mlp.experts.105.gate_proj", "model.layers.59.mlp.experts.106.gate_proj", "model.layers.59.mlp.experts.107.gate_proj", "model.layers.59.mlp.experts.108.gate_proj", "model.layers.59.mlp.experts.109.gate_proj", "model.layers.59.mlp.experts.110.gate_proj", "model.layers.59.mlp.experts.111.gate_proj", "model.layers.59.mlp.experts.112.gate_proj", "model.layers.59.mlp.experts.113.gate_proj", "model.layers.59.mlp.experts.114.gate_proj", "model.layers.59.mlp.experts.115.gate_proj", "model.layers.59.mlp.experts.116.gate_proj", "model.layers.59.mlp.experts.117.gate_proj", "model.layers.59.mlp.experts.118.gate_proj", "model.layers.59.mlp.experts.119.gate_proj", "model.layers.59.mlp.experts.120.gate_proj", "model.layers.59.mlp.experts.121.gate_proj", "model.layers.59.mlp.experts.122.gate_proj", "model.layers.59.mlp.experts.123.gate_proj", "model.layers.59.mlp.experts.124.gate_proj", "model.layers.59.mlp.experts.125.gate_proj", "model.layers.59.mlp.experts.126.gate_proj", "model.layers.59.mlp.experts.127.gate_proj", "model.layers.59.mlp.experts.128.gate_proj", "model.layers.59.mlp.experts.129.gate_proj", "model.layers.59.mlp.experts.130.gate_proj", "model.layers.59.mlp.experts.131.gate_proj", "model.layers.59.mlp.experts.132.gate_proj", "model.layers.59.mlp.experts.133.gate_proj", "model.layers.59.mlp.experts.134.gate_proj", "model.layers.59.mlp.experts.135.gate_proj", "model.layers.59.mlp.experts.136.gate_proj", "model.layers.59.mlp.experts.137.gate_proj", "model.layers.59.mlp.experts.138.gate_proj", "model.layers.59.mlp.experts.139.gate_proj", "model.layers.59.mlp.experts.140.gate_proj", "model.layers.59.mlp.experts.141.gate_proj", "model.layers.59.mlp.experts.142.gate_proj", "model.layers.59.mlp.experts.143.gate_proj", "model.layers.59.mlp.experts.144.gate_proj", "model.layers.59.mlp.experts.145.gate_proj", "model.layers.59.mlp.experts.146.gate_proj", "model.layers.59.mlp.experts.147.gate_proj", "model.layers.59.mlp.experts.148.gate_proj", "model.layers.59.mlp.experts.149.gate_proj", "model.layers.59.mlp.experts.150.gate_proj", "model.layers.59.mlp.experts.151.gate_proj", "model.layers.59.mlp.experts.152.gate_proj", "model.layers.59.mlp.experts.153.gate_proj", "model.layers.59.mlp.experts.154.gate_proj", "model.layers.59.mlp.experts.155.gate_proj", "model.layers.59.mlp.experts.156.gate_proj", "model.layers.59.mlp.experts.157.gate_proj", "model.layers.59.mlp.experts.158.gate_proj", "model.layers.59.mlp.experts.159.gate_proj", "model.layers.59.mlp.experts.0.up_proj", "model.layers.59.mlp.experts.1.up_proj", "model.layers.59.mlp.experts.2.up_proj", "model.layers.59.mlp.experts.3.up_proj", "model.layers.59.mlp.experts.4.up_proj", "model.layers.59.mlp.experts.5.up_proj", "model.layers.59.mlp.experts.6.up_proj", "model.layers.59.mlp.experts.7.up_proj", "model.layers.59.mlp.experts.8.up_proj", "model.layers.59.mlp.experts.9.up_proj", "model.layers.59.mlp.experts.10.up_proj", "model.layers.59.mlp.experts.11.up_proj", "model.layers.59.mlp.experts.12.up_proj", "model.layers.59.mlp.experts.13.up_proj", "model.layers.59.mlp.experts.14.up_proj", "model.layers.59.mlp.experts.15.up_proj", "model.layers.59.mlp.experts.16.up_proj", "model.layers.59.mlp.experts.17.up_proj", "model.layers.59.mlp.experts.18.up_proj", "model.layers.59.mlp.experts.19.up_proj", "model.layers.59.mlp.experts.20.up_proj", "model.layers.59.mlp.experts.21.up_proj", "model.layers.59.mlp.experts.22.up_proj", "model.layers.59.mlp.experts.23.up_proj", "model.layers.59.mlp.experts.24.up_proj", "model.layers.59.mlp.experts.25.up_proj", "model.layers.59.mlp.experts.26.up_proj", "model.layers.59.mlp.experts.27.up_proj", "model.layers.59.mlp.experts.28.up_proj", "model.layers.59.mlp.experts.29.up_proj", "model.layers.59.mlp.experts.30.up_proj", "model.layers.59.mlp.experts.31.up_proj", "model.layers.59.mlp.experts.32.up_proj", "model.layers.59.mlp.experts.33.up_proj", "model.layers.59.mlp.experts.34.up_proj", "model.layers.59.mlp.experts.35.up_proj", "model.layers.59.mlp.experts.36.up_proj", "model.layers.59.mlp.experts.37.up_proj", "model.layers.59.mlp.experts.38.up_proj", "model.layers.59.mlp.experts.39.up_proj", "model.layers.59.mlp.experts.40.up_proj", "model.layers.59.mlp.experts.41.up_proj", "model.layers.59.mlp.experts.42.up_proj", "model.layers.59.mlp.experts.43.up_proj", "model.layers.59.mlp.experts.44.up_proj", "model.layers.59.mlp.experts.45.up_proj", "model.layers.59.mlp.experts.46.up_proj", "model.layers.59.mlp.experts.47.up_proj", "model.layers.59.mlp.experts.48.up_proj", "model.layers.59.mlp.experts.49.up_proj", "model.layers.59.mlp.experts.50.up_proj", "model.layers.59.mlp.experts.51.up_proj", "model.layers.59.mlp.experts.52.up_proj", "model.layers.59.mlp.experts.53.up_proj", "model.layers.59.mlp.experts.54.up_proj", "model.layers.59.mlp.experts.55.up_proj", "model.layers.59.mlp.experts.56.up_proj", "model.layers.59.mlp.experts.57.up_proj", "model.layers.59.mlp.experts.58.up_proj", "model.layers.59.mlp.experts.59.up_proj", "model.layers.59.mlp.experts.60.up_proj", "model.layers.59.mlp.experts.61.up_proj", "model.layers.59.mlp.experts.62.up_proj", "model.layers.59.mlp.experts.63.up_proj", "model.layers.59.mlp.experts.64.up_proj", "model.layers.59.mlp.experts.65.up_proj", "model.layers.59.mlp.experts.66.up_proj", "model.layers.59.mlp.experts.67.up_proj", "model.layers.59.mlp.experts.68.up_proj", "model.layers.59.mlp.experts.69.up_proj", "model.layers.59.mlp.experts.70.up_proj", "model.layers.59.mlp.experts.71.up_proj", "model.layers.59.mlp.experts.72.up_proj", "model.layers.59.mlp.experts.73.up_proj", "model.layers.59.mlp.experts.74.up_proj", "model.layers.59.mlp.experts.75.up_proj", "model.layers.59.mlp.experts.76.up_proj", "model.layers.59.mlp.experts.77.up_proj", "model.layers.59.mlp.experts.78.up_proj", "model.layers.59.mlp.experts.79.up_proj", "model.layers.59.mlp.experts.80.up_proj", "model.layers.59.mlp.experts.81.up_proj", "model.layers.59.mlp.experts.82.up_proj", "model.layers.59.mlp.experts.83.up_proj", "model.layers.59.mlp.experts.84.up_proj", "model.layers.59.mlp.experts.85.up_proj", "model.layers.59.mlp.experts.86.up_proj", "model.layers.59.mlp.experts.87.up_proj", "model.layers.59.mlp.experts.88.up_proj", "model.layers.59.mlp.experts.89.up_proj", "model.layers.59.mlp.experts.90.up_proj", "model.layers.59.mlp.experts.91.up_proj", "model.layers.59.mlp.experts.92.up_proj", "model.layers.59.mlp.experts.93.up_proj", "model.layers.59.mlp.experts.94.up_proj", "model.layers.59.mlp.experts.95.up_proj", "model.layers.59.mlp.experts.96.up_proj", "model.layers.59.mlp.experts.97.up_proj", "model.layers.59.mlp.experts.98.up_proj", "model.layers.59.mlp.experts.99.up_proj", "model.layers.59.mlp.experts.100.up_proj", "model.layers.59.mlp.experts.101.up_proj", "model.layers.59.mlp.experts.102.up_proj", "model.layers.59.mlp.experts.103.up_proj", "model.layers.59.mlp.experts.104.up_proj", "model.layers.59.mlp.experts.105.up_proj", "model.layers.59.mlp.experts.106.up_proj", "model.layers.59.mlp.experts.107.up_proj", "model.layers.59.mlp.experts.108.up_proj", "model.layers.59.mlp.experts.109.up_proj", "model.layers.59.mlp.experts.110.up_proj", "model.layers.59.mlp.experts.111.up_proj", "model.layers.59.mlp.experts.112.up_proj", "model.layers.59.mlp.experts.113.up_proj", "model.layers.59.mlp.experts.114.up_proj", "model.layers.59.mlp.experts.115.up_proj", "model.layers.59.mlp.experts.116.up_proj", "model.layers.59.mlp.experts.117.up_proj", "model.layers.59.mlp.experts.118.up_proj", "model.layers.59.mlp.experts.119.up_proj", "model.layers.59.mlp.experts.120.up_proj", "model.layers.59.mlp.experts.121.up_proj", "model.layers.59.mlp.experts.122.up_proj", "model.layers.59.mlp.experts.123.up_proj", "model.layers.59.mlp.experts.124.up_proj", "model.layers.59.mlp.experts.125.up_proj", "model.layers.59.mlp.experts.126.up_proj", "model.layers.59.mlp.experts.127.up_proj", "model.layers.59.mlp.experts.128.up_proj", "model.layers.59.mlp.experts.129.up_proj", "model.layers.59.mlp.experts.130.up_proj", "model.layers.59.mlp.experts.131.up_proj", "model.layers.59.mlp.experts.132.up_proj", "model.layers.59.mlp.experts.133.up_proj", "model.layers.59.mlp.experts.134.up_proj", "model.layers.59.mlp.experts.135.up_proj", "model.layers.59.mlp.experts.136.up_proj", "model.layers.59.mlp.experts.137.up_proj", "model.layers.59.mlp.experts.138.up_proj", "model.layers.59.mlp.experts.139.up_proj", "model.layers.59.mlp.experts.140.up_proj", "model.layers.59.mlp.experts.141.up_proj", "model.layers.59.mlp.experts.142.up_proj", "model.layers.59.mlp.experts.143.up_proj", "model.layers.59.mlp.experts.144.up_proj", "model.layers.59.mlp.experts.145.up_proj", "model.layers.59.mlp.experts.146.up_proj", "model.layers.59.mlp.experts.147.up_proj", "model.layers.59.mlp.experts.148.up_proj", "model.layers.59.mlp.experts.149.up_proj", "model.layers.59.mlp.experts.150.up_proj", "model.layers.59.mlp.experts.151.up_proj", "model.layers.59.mlp.experts.152.up_proj", "model.layers.59.mlp.experts.153.up_proj", "model.layers.59.mlp.experts.154.up_proj", "model.layers.59.mlp.experts.155.up_proj", "model.layers.59.mlp.experts.156.up_proj", "model.layers.59.mlp.experts.157.up_proj", "model.layers.59.mlp.experts.158.up_proj", "model.layers.59.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.1965457815676928e-06, "dbits": 5033164800 } ] }, { "idx": 356, "layers": [ "model.layers.59.mlp.experts.0.down_proj", "model.layers.59.mlp.experts.1.down_proj", "model.layers.59.mlp.experts.2.down_proj", "model.layers.59.mlp.experts.3.down_proj", "model.layers.59.mlp.experts.4.down_proj", "model.layers.59.mlp.experts.5.down_proj", "model.layers.59.mlp.experts.6.down_proj", "model.layers.59.mlp.experts.7.down_proj", "model.layers.59.mlp.experts.8.down_proj", "model.layers.59.mlp.experts.9.down_proj", "model.layers.59.mlp.experts.10.down_proj", "model.layers.59.mlp.experts.11.down_proj", "model.layers.59.mlp.experts.12.down_proj", "model.layers.59.mlp.experts.13.down_proj", "model.layers.59.mlp.experts.14.down_proj", "model.layers.59.mlp.experts.15.down_proj", "model.layers.59.mlp.experts.16.down_proj", "model.layers.59.mlp.experts.17.down_proj", "model.layers.59.mlp.experts.18.down_proj", "model.layers.59.mlp.experts.19.down_proj", "model.layers.59.mlp.experts.20.down_proj", "model.layers.59.mlp.experts.21.down_proj", "model.layers.59.mlp.experts.22.down_proj", "model.layers.59.mlp.experts.23.down_proj", "model.layers.59.mlp.experts.24.down_proj", "model.layers.59.mlp.experts.25.down_proj", "model.layers.59.mlp.experts.26.down_proj", "model.layers.59.mlp.experts.27.down_proj", "model.layers.59.mlp.experts.28.down_proj", "model.layers.59.mlp.experts.29.down_proj", "model.layers.59.mlp.experts.30.down_proj", "model.layers.59.mlp.experts.31.down_proj", "model.layers.59.mlp.experts.32.down_proj", "model.layers.59.mlp.experts.33.down_proj", "model.layers.59.mlp.experts.34.down_proj", "model.layers.59.mlp.experts.35.down_proj", "model.layers.59.mlp.experts.36.down_proj", "model.layers.59.mlp.experts.37.down_proj", "model.layers.59.mlp.experts.38.down_proj", "model.layers.59.mlp.experts.39.down_proj", "model.layers.59.mlp.experts.40.down_proj", "model.layers.59.mlp.experts.41.down_proj", "model.layers.59.mlp.experts.42.down_proj", "model.layers.59.mlp.experts.43.down_proj", "model.layers.59.mlp.experts.44.down_proj", "model.layers.59.mlp.experts.45.down_proj", "model.layers.59.mlp.experts.46.down_proj", "model.layers.59.mlp.experts.47.down_proj", "model.layers.59.mlp.experts.48.down_proj", "model.layers.59.mlp.experts.49.down_proj", "model.layers.59.mlp.experts.50.down_proj", "model.layers.59.mlp.experts.51.down_proj", "model.layers.59.mlp.experts.52.down_proj", "model.layers.59.mlp.experts.53.down_proj", "model.layers.59.mlp.experts.54.down_proj", "model.layers.59.mlp.experts.55.down_proj", "model.layers.59.mlp.experts.56.down_proj", "model.layers.59.mlp.experts.57.down_proj", "model.layers.59.mlp.experts.58.down_proj", "model.layers.59.mlp.experts.59.down_proj", "model.layers.59.mlp.experts.60.down_proj", "model.layers.59.mlp.experts.61.down_proj", "model.layers.59.mlp.experts.62.down_proj", "model.layers.59.mlp.experts.63.down_proj", "model.layers.59.mlp.experts.64.down_proj", "model.layers.59.mlp.experts.65.down_proj", "model.layers.59.mlp.experts.66.down_proj", "model.layers.59.mlp.experts.67.down_proj", "model.layers.59.mlp.experts.68.down_proj", "model.layers.59.mlp.experts.69.down_proj", "model.layers.59.mlp.experts.70.down_proj", "model.layers.59.mlp.experts.71.down_proj", "model.layers.59.mlp.experts.72.down_proj", "model.layers.59.mlp.experts.73.down_proj", "model.layers.59.mlp.experts.74.down_proj", "model.layers.59.mlp.experts.75.down_proj", "model.layers.59.mlp.experts.76.down_proj", "model.layers.59.mlp.experts.77.down_proj", "model.layers.59.mlp.experts.78.down_proj", "model.layers.59.mlp.experts.79.down_proj", "model.layers.59.mlp.experts.80.down_proj", "model.layers.59.mlp.experts.81.down_proj", "model.layers.59.mlp.experts.82.down_proj", "model.layers.59.mlp.experts.83.down_proj", "model.layers.59.mlp.experts.84.down_proj", "model.layers.59.mlp.experts.85.down_proj", "model.layers.59.mlp.experts.86.down_proj", "model.layers.59.mlp.experts.87.down_proj", "model.layers.59.mlp.experts.88.down_proj", "model.layers.59.mlp.experts.89.down_proj", "model.layers.59.mlp.experts.90.down_proj", "model.layers.59.mlp.experts.91.down_proj", "model.layers.59.mlp.experts.92.down_proj", "model.layers.59.mlp.experts.93.down_proj", "model.layers.59.mlp.experts.94.down_proj", "model.layers.59.mlp.experts.95.down_proj", "model.layers.59.mlp.experts.96.down_proj", "model.layers.59.mlp.experts.97.down_proj", "model.layers.59.mlp.experts.98.down_proj", "model.layers.59.mlp.experts.99.down_proj", "model.layers.59.mlp.experts.100.down_proj", "model.layers.59.mlp.experts.101.down_proj", "model.layers.59.mlp.experts.102.down_proj", "model.layers.59.mlp.experts.103.down_proj", "model.layers.59.mlp.experts.104.down_proj", "model.layers.59.mlp.experts.105.down_proj", "model.layers.59.mlp.experts.106.down_proj", "model.layers.59.mlp.experts.107.down_proj", "model.layers.59.mlp.experts.108.down_proj", "model.layers.59.mlp.experts.109.down_proj", "model.layers.59.mlp.experts.110.down_proj", "model.layers.59.mlp.experts.111.down_proj", "model.layers.59.mlp.experts.112.down_proj", "model.layers.59.mlp.experts.113.down_proj", "model.layers.59.mlp.experts.114.down_proj", "model.layers.59.mlp.experts.115.down_proj", "model.layers.59.mlp.experts.116.down_proj", "model.layers.59.mlp.experts.117.down_proj", "model.layers.59.mlp.experts.118.down_proj", "model.layers.59.mlp.experts.119.down_proj", "model.layers.59.mlp.experts.120.down_proj", "model.layers.59.mlp.experts.121.down_proj", "model.layers.59.mlp.experts.122.down_proj", "model.layers.59.mlp.experts.123.down_proj", "model.layers.59.mlp.experts.124.down_proj", "model.layers.59.mlp.experts.125.down_proj", "model.layers.59.mlp.experts.126.down_proj", "model.layers.59.mlp.experts.127.down_proj", "model.layers.59.mlp.experts.128.down_proj", "model.layers.59.mlp.experts.129.down_proj", "model.layers.59.mlp.experts.130.down_proj", "model.layers.59.mlp.experts.131.down_proj", "model.layers.59.mlp.experts.132.down_proj", "model.layers.59.mlp.experts.133.down_proj", "model.layers.59.mlp.experts.134.down_proj", "model.layers.59.mlp.experts.135.down_proj", "model.layers.59.mlp.experts.136.down_proj", "model.layers.59.mlp.experts.137.down_proj", "model.layers.59.mlp.experts.138.down_proj", "model.layers.59.mlp.experts.139.down_proj", "model.layers.59.mlp.experts.140.down_proj", "model.layers.59.mlp.experts.141.down_proj", "model.layers.59.mlp.experts.142.down_proj", "model.layers.59.mlp.experts.143.down_proj", "model.layers.59.mlp.experts.144.down_proj", "model.layers.59.mlp.experts.145.down_proj", "model.layers.59.mlp.experts.146.down_proj", "model.layers.59.mlp.experts.147.down_proj", "model.layers.59.mlp.experts.148.down_proj", "model.layers.59.mlp.experts.149.down_proj", "model.layers.59.mlp.experts.150.down_proj", "model.layers.59.mlp.experts.151.down_proj", "model.layers.59.mlp.experts.152.down_proj", "model.layers.59.mlp.experts.153.down_proj", "model.layers.59.mlp.experts.154.down_proj", "model.layers.59.mlp.experts.155.down_proj", "model.layers.59.mlp.experts.156.down_proj", "model.layers.59.mlp.experts.157.down_proj", "model.layers.59.mlp.experts.158.down_proj", "model.layers.59.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.4467514120042324e-06, "dbits": 2516582400 } ] }, { "idx": 357, "layers": [ "model.layers.60.self_attn.q_proj" ], "candidates": [ { "dkld": 5.524256266653538e-06, "dbits": 125829120 } ] }, { "idx": 358, "layers": [ "model.layers.60.self_attn.k_proj", "model.layers.60.self_attn.v_proj" ], "candidates": [ { "dkld": 3.1132047297432585e-06, "dbits": 20971520 } ] }, { "idx": 359, "layers": [ "model.layers.60.self_attn.o_proj" ], "candidates": [ { "dkld": 2.453438355587423e-06, "dbits": 125829120 } ] }, { "idx": 360, "layers": [ "model.layers.60.mlp.shared_experts.gate_proj", "model.layers.60.mlp.shared_experts.up_proj", "model.layers.60.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.3955723261460662e-06, "dbits": 47185920 } ] }, { "idx": 361, "layers": [ "model.layers.60.mlp.experts.0.gate_proj", "model.layers.60.mlp.experts.1.gate_proj", "model.layers.60.mlp.experts.2.gate_proj", "model.layers.60.mlp.experts.3.gate_proj", "model.layers.60.mlp.experts.4.gate_proj", "model.layers.60.mlp.experts.5.gate_proj", "model.layers.60.mlp.experts.6.gate_proj", "model.layers.60.mlp.experts.7.gate_proj", "model.layers.60.mlp.experts.8.gate_proj", "model.layers.60.mlp.experts.9.gate_proj", "model.layers.60.mlp.experts.10.gate_proj", "model.layers.60.mlp.experts.11.gate_proj", "model.layers.60.mlp.experts.12.gate_proj", "model.layers.60.mlp.experts.13.gate_proj", "model.layers.60.mlp.experts.14.gate_proj", "model.layers.60.mlp.experts.15.gate_proj", "model.layers.60.mlp.experts.16.gate_proj", "model.layers.60.mlp.experts.17.gate_proj", "model.layers.60.mlp.experts.18.gate_proj", "model.layers.60.mlp.experts.19.gate_proj", "model.layers.60.mlp.experts.20.gate_proj", "model.layers.60.mlp.experts.21.gate_proj", "model.layers.60.mlp.experts.22.gate_proj", "model.layers.60.mlp.experts.23.gate_proj", "model.layers.60.mlp.experts.24.gate_proj", "model.layers.60.mlp.experts.25.gate_proj", "model.layers.60.mlp.experts.26.gate_proj", "model.layers.60.mlp.experts.27.gate_proj", "model.layers.60.mlp.experts.28.gate_proj", "model.layers.60.mlp.experts.29.gate_proj", "model.layers.60.mlp.experts.30.gate_proj", "model.layers.60.mlp.experts.31.gate_proj", "model.layers.60.mlp.experts.32.gate_proj", "model.layers.60.mlp.experts.33.gate_proj", "model.layers.60.mlp.experts.34.gate_proj", "model.layers.60.mlp.experts.35.gate_proj", "model.layers.60.mlp.experts.36.gate_proj", "model.layers.60.mlp.experts.37.gate_proj", "model.layers.60.mlp.experts.38.gate_proj", "model.layers.60.mlp.experts.39.gate_proj", "model.layers.60.mlp.experts.40.gate_proj", "model.layers.60.mlp.experts.41.gate_proj", "model.layers.60.mlp.experts.42.gate_proj", "model.layers.60.mlp.experts.43.gate_proj", "model.layers.60.mlp.experts.44.gate_proj", "model.layers.60.mlp.experts.45.gate_proj", "model.layers.60.mlp.experts.46.gate_proj", "model.layers.60.mlp.experts.47.gate_proj", "model.layers.60.mlp.experts.48.gate_proj", "model.layers.60.mlp.experts.49.gate_proj", "model.layers.60.mlp.experts.50.gate_proj", "model.layers.60.mlp.experts.51.gate_proj", "model.layers.60.mlp.experts.52.gate_proj", "model.layers.60.mlp.experts.53.gate_proj", "model.layers.60.mlp.experts.54.gate_proj", "model.layers.60.mlp.experts.55.gate_proj", "model.layers.60.mlp.experts.56.gate_proj", "model.layers.60.mlp.experts.57.gate_proj", "model.layers.60.mlp.experts.58.gate_proj", "model.layers.60.mlp.experts.59.gate_proj", "model.layers.60.mlp.experts.60.gate_proj", "model.layers.60.mlp.experts.61.gate_proj", "model.layers.60.mlp.experts.62.gate_proj", "model.layers.60.mlp.experts.63.gate_proj", "model.layers.60.mlp.experts.64.gate_proj", "model.layers.60.mlp.experts.65.gate_proj", "model.layers.60.mlp.experts.66.gate_proj", "model.layers.60.mlp.experts.67.gate_proj", "model.layers.60.mlp.experts.68.gate_proj", "model.layers.60.mlp.experts.69.gate_proj", "model.layers.60.mlp.experts.70.gate_proj", "model.layers.60.mlp.experts.71.gate_proj", "model.layers.60.mlp.experts.72.gate_proj", "model.layers.60.mlp.experts.73.gate_proj", "model.layers.60.mlp.experts.74.gate_proj", "model.layers.60.mlp.experts.75.gate_proj", "model.layers.60.mlp.experts.76.gate_proj", "model.layers.60.mlp.experts.77.gate_proj", "model.layers.60.mlp.experts.78.gate_proj", "model.layers.60.mlp.experts.79.gate_proj", "model.layers.60.mlp.experts.80.gate_proj", "model.layers.60.mlp.experts.81.gate_proj", "model.layers.60.mlp.experts.82.gate_proj", "model.layers.60.mlp.experts.83.gate_proj", "model.layers.60.mlp.experts.84.gate_proj", "model.layers.60.mlp.experts.85.gate_proj", "model.layers.60.mlp.experts.86.gate_proj", "model.layers.60.mlp.experts.87.gate_proj", "model.layers.60.mlp.experts.88.gate_proj", "model.layers.60.mlp.experts.89.gate_proj", "model.layers.60.mlp.experts.90.gate_proj", "model.layers.60.mlp.experts.91.gate_proj", "model.layers.60.mlp.experts.92.gate_proj", "model.layers.60.mlp.experts.93.gate_proj", "model.layers.60.mlp.experts.94.gate_proj", "model.layers.60.mlp.experts.95.gate_proj", "model.layers.60.mlp.experts.96.gate_proj", "model.layers.60.mlp.experts.97.gate_proj", "model.layers.60.mlp.experts.98.gate_proj", "model.layers.60.mlp.experts.99.gate_proj", "model.layers.60.mlp.experts.100.gate_proj", "model.layers.60.mlp.experts.101.gate_proj", "model.layers.60.mlp.experts.102.gate_proj", "model.layers.60.mlp.experts.103.gate_proj", "model.layers.60.mlp.experts.104.gate_proj", "model.layers.60.mlp.experts.105.gate_proj", "model.layers.60.mlp.experts.106.gate_proj", "model.layers.60.mlp.experts.107.gate_proj", "model.layers.60.mlp.experts.108.gate_proj", "model.layers.60.mlp.experts.109.gate_proj", "model.layers.60.mlp.experts.110.gate_proj", "model.layers.60.mlp.experts.111.gate_proj", "model.layers.60.mlp.experts.112.gate_proj", "model.layers.60.mlp.experts.113.gate_proj", "model.layers.60.mlp.experts.114.gate_proj", "model.layers.60.mlp.experts.115.gate_proj", "model.layers.60.mlp.experts.116.gate_proj", "model.layers.60.mlp.experts.117.gate_proj", "model.layers.60.mlp.experts.118.gate_proj", "model.layers.60.mlp.experts.119.gate_proj", "model.layers.60.mlp.experts.120.gate_proj", "model.layers.60.mlp.experts.121.gate_proj", "model.layers.60.mlp.experts.122.gate_proj", "model.layers.60.mlp.experts.123.gate_proj", "model.layers.60.mlp.experts.124.gate_proj", "model.layers.60.mlp.experts.125.gate_proj", "model.layers.60.mlp.experts.126.gate_proj", "model.layers.60.mlp.experts.127.gate_proj", "model.layers.60.mlp.experts.128.gate_proj", "model.layers.60.mlp.experts.129.gate_proj", "model.layers.60.mlp.experts.130.gate_proj", "model.layers.60.mlp.experts.131.gate_proj", "model.layers.60.mlp.experts.132.gate_proj", "model.layers.60.mlp.experts.133.gate_proj", "model.layers.60.mlp.experts.134.gate_proj", "model.layers.60.mlp.experts.135.gate_proj", "model.layers.60.mlp.experts.136.gate_proj", "model.layers.60.mlp.experts.137.gate_proj", "model.layers.60.mlp.experts.138.gate_proj", "model.layers.60.mlp.experts.139.gate_proj", "model.layers.60.mlp.experts.140.gate_proj", "model.layers.60.mlp.experts.141.gate_proj", "model.layers.60.mlp.experts.142.gate_proj", "model.layers.60.mlp.experts.143.gate_proj", "model.layers.60.mlp.experts.144.gate_proj", "model.layers.60.mlp.experts.145.gate_proj", "model.layers.60.mlp.experts.146.gate_proj", "model.layers.60.mlp.experts.147.gate_proj", "model.layers.60.mlp.experts.148.gate_proj", "model.layers.60.mlp.experts.149.gate_proj", "model.layers.60.mlp.experts.150.gate_proj", "model.layers.60.mlp.experts.151.gate_proj", "model.layers.60.mlp.experts.152.gate_proj", "model.layers.60.mlp.experts.153.gate_proj", "model.layers.60.mlp.experts.154.gate_proj", "model.layers.60.mlp.experts.155.gate_proj", "model.layers.60.mlp.experts.156.gate_proj", "model.layers.60.mlp.experts.157.gate_proj", "model.layers.60.mlp.experts.158.gate_proj", "model.layers.60.mlp.experts.159.gate_proj", "model.layers.60.mlp.experts.0.up_proj", "model.layers.60.mlp.experts.1.up_proj", "model.layers.60.mlp.experts.2.up_proj", "model.layers.60.mlp.experts.3.up_proj", "model.layers.60.mlp.experts.4.up_proj", "model.layers.60.mlp.experts.5.up_proj", "model.layers.60.mlp.experts.6.up_proj", "model.layers.60.mlp.experts.7.up_proj", "model.layers.60.mlp.experts.8.up_proj", "model.layers.60.mlp.experts.9.up_proj", "model.layers.60.mlp.experts.10.up_proj", "model.layers.60.mlp.experts.11.up_proj", "model.layers.60.mlp.experts.12.up_proj", "model.layers.60.mlp.experts.13.up_proj", "model.layers.60.mlp.experts.14.up_proj", "model.layers.60.mlp.experts.15.up_proj", "model.layers.60.mlp.experts.16.up_proj", "model.layers.60.mlp.experts.17.up_proj", "model.layers.60.mlp.experts.18.up_proj", "model.layers.60.mlp.experts.19.up_proj", "model.layers.60.mlp.experts.20.up_proj", "model.layers.60.mlp.experts.21.up_proj", "model.layers.60.mlp.experts.22.up_proj", "model.layers.60.mlp.experts.23.up_proj", "model.layers.60.mlp.experts.24.up_proj", "model.layers.60.mlp.experts.25.up_proj", "model.layers.60.mlp.experts.26.up_proj", "model.layers.60.mlp.experts.27.up_proj", "model.layers.60.mlp.experts.28.up_proj", "model.layers.60.mlp.experts.29.up_proj", "model.layers.60.mlp.experts.30.up_proj", "model.layers.60.mlp.experts.31.up_proj", "model.layers.60.mlp.experts.32.up_proj", "model.layers.60.mlp.experts.33.up_proj", "model.layers.60.mlp.experts.34.up_proj", "model.layers.60.mlp.experts.35.up_proj", "model.layers.60.mlp.experts.36.up_proj", "model.layers.60.mlp.experts.37.up_proj", "model.layers.60.mlp.experts.38.up_proj", "model.layers.60.mlp.experts.39.up_proj", "model.layers.60.mlp.experts.40.up_proj", "model.layers.60.mlp.experts.41.up_proj", "model.layers.60.mlp.experts.42.up_proj", "model.layers.60.mlp.experts.43.up_proj", "model.layers.60.mlp.experts.44.up_proj", "model.layers.60.mlp.experts.45.up_proj", "model.layers.60.mlp.experts.46.up_proj", "model.layers.60.mlp.experts.47.up_proj", "model.layers.60.mlp.experts.48.up_proj", "model.layers.60.mlp.experts.49.up_proj", "model.layers.60.mlp.experts.50.up_proj", "model.layers.60.mlp.experts.51.up_proj", "model.layers.60.mlp.experts.52.up_proj", "model.layers.60.mlp.experts.53.up_proj", "model.layers.60.mlp.experts.54.up_proj", "model.layers.60.mlp.experts.55.up_proj", "model.layers.60.mlp.experts.56.up_proj", "model.layers.60.mlp.experts.57.up_proj", "model.layers.60.mlp.experts.58.up_proj", "model.layers.60.mlp.experts.59.up_proj", "model.layers.60.mlp.experts.60.up_proj", "model.layers.60.mlp.experts.61.up_proj", "model.layers.60.mlp.experts.62.up_proj", "model.layers.60.mlp.experts.63.up_proj", "model.layers.60.mlp.experts.64.up_proj", "model.layers.60.mlp.experts.65.up_proj", "model.layers.60.mlp.experts.66.up_proj", "model.layers.60.mlp.experts.67.up_proj", "model.layers.60.mlp.experts.68.up_proj", "model.layers.60.mlp.experts.69.up_proj", "model.layers.60.mlp.experts.70.up_proj", "model.layers.60.mlp.experts.71.up_proj", "model.layers.60.mlp.experts.72.up_proj", "model.layers.60.mlp.experts.73.up_proj", "model.layers.60.mlp.experts.74.up_proj", "model.layers.60.mlp.experts.75.up_proj", "model.layers.60.mlp.experts.76.up_proj", "model.layers.60.mlp.experts.77.up_proj", "model.layers.60.mlp.experts.78.up_proj", "model.layers.60.mlp.experts.79.up_proj", "model.layers.60.mlp.experts.80.up_proj", "model.layers.60.mlp.experts.81.up_proj", "model.layers.60.mlp.experts.82.up_proj", "model.layers.60.mlp.experts.83.up_proj", "model.layers.60.mlp.experts.84.up_proj", "model.layers.60.mlp.experts.85.up_proj", "model.layers.60.mlp.experts.86.up_proj", "model.layers.60.mlp.experts.87.up_proj", "model.layers.60.mlp.experts.88.up_proj", "model.layers.60.mlp.experts.89.up_proj", "model.layers.60.mlp.experts.90.up_proj", "model.layers.60.mlp.experts.91.up_proj", "model.layers.60.mlp.experts.92.up_proj", "model.layers.60.mlp.experts.93.up_proj", "model.layers.60.mlp.experts.94.up_proj", "model.layers.60.mlp.experts.95.up_proj", "model.layers.60.mlp.experts.96.up_proj", "model.layers.60.mlp.experts.97.up_proj", "model.layers.60.mlp.experts.98.up_proj", "model.layers.60.mlp.experts.99.up_proj", "model.layers.60.mlp.experts.100.up_proj", "model.layers.60.mlp.experts.101.up_proj", "model.layers.60.mlp.experts.102.up_proj", "model.layers.60.mlp.experts.103.up_proj", "model.layers.60.mlp.experts.104.up_proj", "model.layers.60.mlp.experts.105.up_proj", "model.layers.60.mlp.experts.106.up_proj", "model.layers.60.mlp.experts.107.up_proj", "model.layers.60.mlp.experts.108.up_proj", "model.layers.60.mlp.experts.109.up_proj", "model.layers.60.mlp.experts.110.up_proj", "model.layers.60.mlp.experts.111.up_proj", "model.layers.60.mlp.experts.112.up_proj", "model.layers.60.mlp.experts.113.up_proj", "model.layers.60.mlp.experts.114.up_proj", "model.layers.60.mlp.experts.115.up_proj", "model.layers.60.mlp.experts.116.up_proj", "model.layers.60.mlp.experts.117.up_proj", "model.layers.60.mlp.experts.118.up_proj", "model.layers.60.mlp.experts.119.up_proj", "model.layers.60.mlp.experts.120.up_proj", "model.layers.60.mlp.experts.121.up_proj", "model.layers.60.mlp.experts.122.up_proj", "model.layers.60.mlp.experts.123.up_proj", "model.layers.60.mlp.experts.124.up_proj", "model.layers.60.mlp.experts.125.up_proj", "model.layers.60.mlp.experts.126.up_proj", "model.layers.60.mlp.experts.127.up_proj", "model.layers.60.mlp.experts.128.up_proj", "model.layers.60.mlp.experts.129.up_proj", "model.layers.60.mlp.experts.130.up_proj", "model.layers.60.mlp.experts.131.up_proj", "model.layers.60.mlp.experts.132.up_proj", "model.layers.60.mlp.experts.133.up_proj", "model.layers.60.mlp.experts.134.up_proj", "model.layers.60.mlp.experts.135.up_proj", "model.layers.60.mlp.experts.136.up_proj", "model.layers.60.mlp.experts.137.up_proj", "model.layers.60.mlp.experts.138.up_proj", "model.layers.60.mlp.experts.139.up_proj", "model.layers.60.mlp.experts.140.up_proj", "model.layers.60.mlp.experts.141.up_proj", "model.layers.60.mlp.experts.142.up_proj", "model.layers.60.mlp.experts.143.up_proj", "model.layers.60.mlp.experts.144.up_proj", "model.layers.60.mlp.experts.145.up_proj", "model.layers.60.mlp.experts.146.up_proj", "model.layers.60.mlp.experts.147.up_proj", "model.layers.60.mlp.experts.148.up_proj", "model.layers.60.mlp.experts.149.up_proj", "model.layers.60.mlp.experts.150.up_proj", "model.layers.60.mlp.experts.151.up_proj", "model.layers.60.mlp.experts.152.up_proj", "model.layers.60.mlp.experts.153.up_proj", "model.layers.60.mlp.experts.154.up_proj", "model.layers.60.mlp.experts.155.up_proj", "model.layers.60.mlp.experts.156.up_proj", "model.layers.60.mlp.experts.157.up_proj", "model.layers.60.mlp.experts.158.up_proj", "model.layers.60.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.3541983207687302e-06, "dbits": 5033164800 } ] }, { "idx": 362, "layers": [ "model.layers.60.mlp.experts.0.down_proj", "model.layers.60.mlp.experts.1.down_proj", "model.layers.60.mlp.experts.2.down_proj", "model.layers.60.mlp.experts.3.down_proj", "model.layers.60.mlp.experts.4.down_proj", "model.layers.60.mlp.experts.5.down_proj", "model.layers.60.mlp.experts.6.down_proj", "model.layers.60.mlp.experts.7.down_proj", "model.layers.60.mlp.experts.8.down_proj", "model.layers.60.mlp.experts.9.down_proj", "model.layers.60.mlp.experts.10.down_proj", "model.layers.60.mlp.experts.11.down_proj", "model.layers.60.mlp.experts.12.down_proj", "model.layers.60.mlp.experts.13.down_proj", "model.layers.60.mlp.experts.14.down_proj", "model.layers.60.mlp.experts.15.down_proj", "model.layers.60.mlp.experts.16.down_proj", "model.layers.60.mlp.experts.17.down_proj", "model.layers.60.mlp.experts.18.down_proj", "model.layers.60.mlp.experts.19.down_proj", "model.layers.60.mlp.experts.20.down_proj", "model.layers.60.mlp.experts.21.down_proj", "model.layers.60.mlp.experts.22.down_proj", "model.layers.60.mlp.experts.23.down_proj", "model.layers.60.mlp.experts.24.down_proj", "model.layers.60.mlp.experts.25.down_proj", "model.layers.60.mlp.experts.26.down_proj", "model.layers.60.mlp.experts.27.down_proj", "model.layers.60.mlp.experts.28.down_proj", "model.layers.60.mlp.experts.29.down_proj", "model.layers.60.mlp.experts.30.down_proj", "model.layers.60.mlp.experts.31.down_proj", "model.layers.60.mlp.experts.32.down_proj", "model.layers.60.mlp.experts.33.down_proj", "model.layers.60.mlp.experts.34.down_proj", "model.layers.60.mlp.experts.35.down_proj", "model.layers.60.mlp.experts.36.down_proj", "model.layers.60.mlp.experts.37.down_proj", "model.layers.60.mlp.experts.38.down_proj", "model.layers.60.mlp.experts.39.down_proj", "model.layers.60.mlp.experts.40.down_proj", "model.layers.60.mlp.experts.41.down_proj", "model.layers.60.mlp.experts.42.down_proj", "model.layers.60.mlp.experts.43.down_proj", "model.layers.60.mlp.experts.44.down_proj", "model.layers.60.mlp.experts.45.down_proj", "model.layers.60.mlp.experts.46.down_proj", "model.layers.60.mlp.experts.47.down_proj", "model.layers.60.mlp.experts.48.down_proj", "model.layers.60.mlp.experts.49.down_proj", "model.layers.60.mlp.experts.50.down_proj", "model.layers.60.mlp.experts.51.down_proj", "model.layers.60.mlp.experts.52.down_proj", "model.layers.60.mlp.experts.53.down_proj", "model.layers.60.mlp.experts.54.down_proj", "model.layers.60.mlp.experts.55.down_proj", "model.layers.60.mlp.experts.56.down_proj", "model.layers.60.mlp.experts.57.down_proj", "model.layers.60.mlp.experts.58.down_proj", "model.layers.60.mlp.experts.59.down_proj", "model.layers.60.mlp.experts.60.down_proj", "model.layers.60.mlp.experts.61.down_proj", "model.layers.60.mlp.experts.62.down_proj", "model.layers.60.mlp.experts.63.down_proj", "model.layers.60.mlp.experts.64.down_proj", "model.layers.60.mlp.experts.65.down_proj", "model.layers.60.mlp.experts.66.down_proj", "model.layers.60.mlp.experts.67.down_proj", "model.layers.60.mlp.experts.68.down_proj", "model.layers.60.mlp.experts.69.down_proj", "model.layers.60.mlp.experts.70.down_proj", "model.layers.60.mlp.experts.71.down_proj", "model.layers.60.mlp.experts.72.down_proj", "model.layers.60.mlp.experts.73.down_proj", "model.layers.60.mlp.experts.74.down_proj", "model.layers.60.mlp.experts.75.down_proj", "model.layers.60.mlp.experts.76.down_proj", "model.layers.60.mlp.experts.77.down_proj", "model.layers.60.mlp.experts.78.down_proj", "model.layers.60.mlp.experts.79.down_proj", "model.layers.60.mlp.experts.80.down_proj", "model.layers.60.mlp.experts.81.down_proj", "model.layers.60.mlp.experts.82.down_proj", "model.layers.60.mlp.experts.83.down_proj", "model.layers.60.mlp.experts.84.down_proj", "model.layers.60.mlp.experts.85.down_proj", "model.layers.60.mlp.experts.86.down_proj", "model.layers.60.mlp.experts.87.down_proj", "model.layers.60.mlp.experts.88.down_proj", "model.layers.60.mlp.experts.89.down_proj", "model.layers.60.mlp.experts.90.down_proj", "model.layers.60.mlp.experts.91.down_proj", "model.layers.60.mlp.experts.92.down_proj", "model.layers.60.mlp.experts.93.down_proj", "model.layers.60.mlp.experts.94.down_proj", "model.layers.60.mlp.experts.95.down_proj", "model.layers.60.mlp.experts.96.down_proj", "model.layers.60.mlp.experts.97.down_proj", "model.layers.60.mlp.experts.98.down_proj", "model.layers.60.mlp.experts.99.down_proj", "model.layers.60.mlp.experts.100.down_proj", "model.layers.60.mlp.experts.101.down_proj", "model.layers.60.mlp.experts.102.down_proj", "model.layers.60.mlp.experts.103.down_proj", "model.layers.60.mlp.experts.104.down_proj", "model.layers.60.mlp.experts.105.down_proj", "model.layers.60.mlp.experts.106.down_proj", "model.layers.60.mlp.experts.107.down_proj", "model.layers.60.mlp.experts.108.down_proj", "model.layers.60.mlp.experts.109.down_proj", "model.layers.60.mlp.experts.110.down_proj", "model.layers.60.mlp.experts.111.down_proj", "model.layers.60.mlp.experts.112.down_proj", "model.layers.60.mlp.experts.113.down_proj", "model.layers.60.mlp.experts.114.down_proj", "model.layers.60.mlp.experts.115.down_proj", "model.layers.60.mlp.experts.116.down_proj", "model.layers.60.mlp.experts.117.down_proj", "model.layers.60.mlp.experts.118.down_proj", "model.layers.60.mlp.experts.119.down_proj", "model.layers.60.mlp.experts.120.down_proj", "model.layers.60.mlp.experts.121.down_proj", "model.layers.60.mlp.experts.122.down_proj", "model.layers.60.mlp.experts.123.down_proj", "model.layers.60.mlp.experts.124.down_proj", "model.layers.60.mlp.experts.125.down_proj", "model.layers.60.mlp.experts.126.down_proj", "model.layers.60.mlp.experts.127.down_proj", "model.layers.60.mlp.experts.128.down_proj", "model.layers.60.mlp.experts.129.down_proj", "model.layers.60.mlp.experts.130.down_proj", "model.layers.60.mlp.experts.131.down_proj", "model.layers.60.mlp.experts.132.down_proj", "model.layers.60.mlp.experts.133.down_proj", "model.layers.60.mlp.experts.134.down_proj", "model.layers.60.mlp.experts.135.down_proj", "model.layers.60.mlp.experts.136.down_proj", "model.layers.60.mlp.experts.137.down_proj", "model.layers.60.mlp.experts.138.down_proj", "model.layers.60.mlp.experts.139.down_proj", "model.layers.60.mlp.experts.140.down_proj", "model.layers.60.mlp.experts.141.down_proj", "model.layers.60.mlp.experts.142.down_proj", "model.layers.60.mlp.experts.143.down_proj", "model.layers.60.mlp.experts.144.down_proj", "model.layers.60.mlp.experts.145.down_proj", "model.layers.60.mlp.experts.146.down_proj", "model.layers.60.mlp.experts.147.down_proj", "model.layers.60.mlp.experts.148.down_proj", "model.layers.60.mlp.experts.149.down_proj", "model.layers.60.mlp.experts.150.down_proj", "model.layers.60.mlp.experts.151.down_proj", "model.layers.60.mlp.experts.152.down_proj", "model.layers.60.mlp.experts.153.down_proj", "model.layers.60.mlp.experts.154.down_proj", "model.layers.60.mlp.experts.155.down_proj", "model.layers.60.mlp.experts.156.down_proj", "model.layers.60.mlp.experts.157.down_proj", "model.layers.60.mlp.experts.158.down_proj", "model.layers.60.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.8395239496603825e-06, "dbits": 2516582400 } ] }, { "idx": 363, "layers": [ "model.layers.61.self_attn.q_proj" ], "candidates": [ { "dkld": 4.326942143961831e-06, "dbits": 125829120 } ] }, { "idx": 364, "layers": [ "model.layers.61.self_attn.k_proj", "model.layers.61.self_attn.v_proj" ], "candidates": [ { "dkld": -1.2350006727501967e-06, "dbits": 20971520 } ] }, { "idx": 365, "layers": [ "model.layers.61.self_attn.o_proj" ], "candidates": [ { "dkld": -8.955481462180831e-07, "dbits": 125829120 } ] }, { "idx": 366, "layers": [ "model.layers.61.mlp.shared_experts.gate_proj", "model.layers.61.mlp.shared_experts.up_proj", "model.layers.61.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.888811286538861e-06, "dbits": 47185920 } ] }, { "idx": 367, "layers": [ "model.layers.61.mlp.experts.0.gate_proj", "model.layers.61.mlp.experts.1.gate_proj", "model.layers.61.mlp.experts.2.gate_proj", "model.layers.61.mlp.experts.3.gate_proj", "model.layers.61.mlp.experts.4.gate_proj", "model.layers.61.mlp.experts.5.gate_proj", "model.layers.61.mlp.experts.6.gate_proj", "model.layers.61.mlp.experts.7.gate_proj", "model.layers.61.mlp.experts.8.gate_proj", "model.layers.61.mlp.experts.9.gate_proj", "model.layers.61.mlp.experts.10.gate_proj", "model.layers.61.mlp.experts.11.gate_proj", "model.layers.61.mlp.experts.12.gate_proj", "model.layers.61.mlp.experts.13.gate_proj", "model.layers.61.mlp.experts.14.gate_proj", "model.layers.61.mlp.experts.15.gate_proj", "model.layers.61.mlp.experts.16.gate_proj", "model.layers.61.mlp.experts.17.gate_proj", "model.layers.61.mlp.experts.18.gate_proj", "model.layers.61.mlp.experts.19.gate_proj", "model.layers.61.mlp.experts.20.gate_proj", "model.layers.61.mlp.experts.21.gate_proj", "model.layers.61.mlp.experts.22.gate_proj", "model.layers.61.mlp.experts.23.gate_proj", "model.layers.61.mlp.experts.24.gate_proj", "model.layers.61.mlp.experts.25.gate_proj", "model.layers.61.mlp.experts.26.gate_proj", "model.layers.61.mlp.experts.27.gate_proj", "model.layers.61.mlp.experts.28.gate_proj", "model.layers.61.mlp.experts.29.gate_proj", "model.layers.61.mlp.experts.30.gate_proj", "model.layers.61.mlp.experts.31.gate_proj", "model.layers.61.mlp.experts.32.gate_proj", "model.layers.61.mlp.experts.33.gate_proj", "model.layers.61.mlp.experts.34.gate_proj", "model.layers.61.mlp.experts.35.gate_proj", "model.layers.61.mlp.experts.36.gate_proj", "model.layers.61.mlp.experts.37.gate_proj", "model.layers.61.mlp.experts.38.gate_proj", "model.layers.61.mlp.experts.39.gate_proj", "model.layers.61.mlp.experts.40.gate_proj", "model.layers.61.mlp.experts.41.gate_proj", "model.layers.61.mlp.experts.42.gate_proj", "model.layers.61.mlp.experts.43.gate_proj", "model.layers.61.mlp.experts.44.gate_proj", "model.layers.61.mlp.experts.45.gate_proj", "model.layers.61.mlp.experts.46.gate_proj", "model.layers.61.mlp.experts.47.gate_proj", "model.layers.61.mlp.experts.48.gate_proj", "model.layers.61.mlp.experts.49.gate_proj", "model.layers.61.mlp.experts.50.gate_proj", "model.layers.61.mlp.experts.51.gate_proj", "model.layers.61.mlp.experts.52.gate_proj", "model.layers.61.mlp.experts.53.gate_proj", "model.layers.61.mlp.experts.54.gate_proj", "model.layers.61.mlp.experts.55.gate_proj", "model.layers.61.mlp.experts.56.gate_proj", "model.layers.61.mlp.experts.57.gate_proj", "model.layers.61.mlp.experts.58.gate_proj", "model.layers.61.mlp.experts.59.gate_proj", "model.layers.61.mlp.experts.60.gate_proj", "model.layers.61.mlp.experts.61.gate_proj", "model.layers.61.mlp.experts.62.gate_proj", "model.layers.61.mlp.experts.63.gate_proj", "model.layers.61.mlp.experts.64.gate_proj", "model.layers.61.mlp.experts.65.gate_proj", "model.layers.61.mlp.experts.66.gate_proj", "model.layers.61.mlp.experts.67.gate_proj", "model.layers.61.mlp.experts.68.gate_proj", "model.layers.61.mlp.experts.69.gate_proj", "model.layers.61.mlp.experts.70.gate_proj", "model.layers.61.mlp.experts.71.gate_proj", "model.layers.61.mlp.experts.72.gate_proj", "model.layers.61.mlp.experts.73.gate_proj", "model.layers.61.mlp.experts.74.gate_proj", "model.layers.61.mlp.experts.75.gate_proj", "model.layers.61.mlp.experts.76.gate_proj", "model.layers.61.mlp.experts.77.gate_proj", "model.layers.61.mlp.experts.78.gate_proj", "model.layers.61.mlp.experts.79.gate_proj", "model.layers.61.mlp.experts.80.gate_proj", "model.layers.61.mlp.experts.81.gate_proj", "model.layers.61.mlp.experts.82.gate_proj", "model.layers.61.mlp.experts.83.gate_proj", "model.layers.61.mlp.experts.84.gate_proj", "model.layers.61.mlp.experts.85.gate_proj", "model.layers.61.mlp.experts.86.gate_proj", "model.layers.61.mlp.experts.87.gate_proj", "model.layers.61.mlp.experts.88.gate_proj", "model.layers.61.mlp.experts.89.gate_proj", "model.layers.61.mlp.experts.90.gate_proj", "model.layers.61.mlp.experts.91.gate_proj", "model.layers.61.mlp.experts.92.gate_proj", "model.layers.61.mlp.experts.93.gate_proj", "model.layers.61.mlp.experts.94.gate_proj", "model.layers.61.mlp.experts.95.gate_proj", "model.layers.61.mlp.experts.96.gate_proj", "model.layers.61.mlp.experts.97.gate_proj", "model.layers.61.mlp.experts.98.gate_proj", "model.layers.61.mlp.experts.99.gate_proj", "model.layers.61.mlp.experts.100.gate_proj", "model.layers.61.mlp.experts.101.gate_proj", "model.layers.61.mlp.experts.102.gate_proj", "model.layers.61.mlp.experts.103.gate_proj", "model.layers.61.mlp.experts.104.gate_proj", "model.layers.61.mlp.experts.105.gate_proj", "model.layers.61.mlp.experts.106.gate_proj", "model.layers.61.mlp.experts.107.gate_proj", "model.layers.61.mlp.experts.108.gate_proj", "model.layers.61.mlp.experts.109.gate_proj", "model.layers.61.mlp.experts.110.gate_proj", "model.layers.61.mlp.experts.111.gate_proj", "model.layers.61.mlp.experts.112.gate_proj", "model.layers.61.mlp.experts.113.gate_proj", "model.layers.61.mlp.experts.114.gate_proj", "model.layers.61.mlp.experts.115.gate_proj", "model.layers.61.mlp.experts.116.gate_proj", "model.layers.61.mlp.experts.117.gate_proj", "model.layers.61.mlp.experts.118.gate_proj", "model.layers.61.mlp.experts.119.gate_proj", "model.layers.61.mlp.experts.120.gate_proj", "model.layers.61.mlp.experts.121.gate_proj", "model.layers.61.mlp.experts.122.gate_proj", "model.layers.61.mlp.experts.123.gate_proj", "model.layers.61.mlp.experts.124.gate_proj", "model.layers.61.mlp.experts.125.gate_proj", "model.layers.61.mlp.experts.126.gate_proj", "model.layers.61.mlp.experts.127.gate_proj", "model.layers.61.mlp.experts.128.gate_proj", "model.layers.61.mlp.experts.129.gate_proj", "model.layers.61.mlp.experts.130.gate_proj", "model.layers.61.mlp.experts.131.gate_proj", "model.layers.61.mlp.experts.132.gate_proj", "model.layers.61.mlp.experts.133.gate_proj", "model.layers.61.mlp.experts.134.gate_proj", "model.layers.61.mlp.experts.135.gate_proj", "model.layers.61.mlp.experts.136.gate_proj", "model.layers.61.mlp.experts.137.gate_proj", "model.layers.61.mlp.experts.138.gate_proj", "model.layers.61.mlp.experts.139.gate_proj", "model.layers.61.mlp.experts.140.gate_proj", "model.layers.61.mlp.experts.141.gate_proj", "model.layers.61.mlp.experts.142.gate_proj", "model.layers.61.mlp.experts.143.gate_proj", "model.layers.61.mlp.experts.144.gate_proj", "model.layers.61.mlp.experts.145.gate_proj", "model.layers.61.mlp.experts.146.gate_proj", "model.layers.61.mlp.experts.147.gate_proj", "model.layers.61.mlp.experts.148.gate_proj", "model.layers.61.mlp.experts.149.gate_proj", "model.layers.61.mlp.experts.150.gate_proj", "model.layers.61.mlp.experts.151.gate_proj", "model.layers.61.mlp.experts.152.gate_proj", "model.layers.61.mlp.experts.153.gate_proj", "model.layers.61.mlp.experts.154.gate_proj", "model.layers.61.mlp.experts.155.gate_proj", "model.layers.61.mlp.experts.156.gate_proj", "model.layers.61.mlp.experts.157.gate_proj", "model.layers.61.mlp.experts.158.gate_proj", "model.layers.61.mlp.experts.159.gate_proj", "model.layers.61.mlp.experts.0.up_proj", "model.layers.61.mlp.experts.1.up_proj", "model.layers.61.mlp.experts.2.up_proj", "model.layers.61.mlp.experts.3.up_proj", "model.layers.61.mlp.experts.4.up_proj", "model.layers.61.mlp.experts.5.up_proj", "model.layers.61.mlp.experts.6.up_proj", "model.layers.61.mlp.experts.7.up_proj", "model.layers.61.mlp.experts.8.up_proj", "model.layers.61.mlp.experts.9.up_proj", "model.layers.61.mlp.experts.10.up_proj", "model.layers.61.mlp.experts.11.up_proj", "model.layers.61.mlp.experts.12.up_proj", "model.layers.61.mlp.experts.13.up_proj", "model.layers.61.mlp.experts.14.up_proj", "model.layers.61.mlp.experts.15.up_proj", "model.layers.61.mlp.experts.16.up_proj", "model.layers.61.mlp.experts.17.up_proj", "model.layers.61.mlp.experts.18.up_proj", "model.layers.61.mlp.experts.19.up_proj", "model.layers.61.mlp.experts.20.up_proj", "model.layers.61.mlp.experts.21.up_proj", "model.layers.61.mlp.experts.22.up_proj", "model.layers.61.mlp.experts.23.up_proj", "model.layers.61.mlp.experts.24.up_proj", "model.layers.61.mlp.experts.25.up_proj", "model.layers.61.mlp.experts.26.up_proj", "model.layers.61.mlp.experts.27.up_proj", "model.layers.61.mlp.experts.28.up_proj", "model.layers.61.mlp.experts.29.up_proj", "model.layers.61.mlp.experts.30.up_proj", "model.layers.61.mlp.experts.31.up_proj", "model.layers.61.mlp.experts.32.up_proj", "model.layers.61.mlp.experts.33.up_proj", "model.layers.61.mlp.experts.34.up_proj", "model.layers.61.mlp.experts.35.up_proj", "model.layers.61.mlp.experts.36.up_proj", "model.layers.61.mlp.experts.37.up_proj", "model.layers.61.mlp.experts.38.up_proj", "model.layers.61.mlp.experts.39.up_proj", "model.layers.61.mlp.experts.40.up_proj", "model.layers.61.mlp.experts.41.up_proj", "model.layers.61.mlp.experts.42.up_proj", "model.layers.61.mlp.experts.43.up_proj", "model.layers.61.mlp.experts.44.up_proj", "model.layers.61.mlp.experts.45.up_proj", "model.layers.61.mlp.experts.46.up_proj", "model.layers.61.mlp.experts.47.up_proj", "model.layers.61.mlp.experts.48.up_proj", "model.layers.61.mlp.experts.49.up_proj", "model.layers.61.mlp.experts.50.up_proj", "model.layers.61.mlp.experts.51.up_proj", "model.layers.61.mlp.experts.52.up_proj", "model.layers.61.mlp.experts.53.up_proj", "model.layers.61.mlp.experts.54.up_proj", "model.layers.61.mlp.experts.55.up_proj", "model.layers.61.mlp.experts.56.up_proj", "model.layers.61.mlp.experts.57.up_proj", "model.layers.61.mlp.experts.58.up_proj", "model.layers.61.mlp.experts.59.up_proj", "model.layers.61.mlp.experts.60.up_proj", "model.layers.61.mlp.experts.61.up_proj", "model.layers.61.mlp.experts.62.up_proj", "model.layers.61.mlp.experts.63.up_proj", "model.layers.61.mlp.experts.64.up_proj", "model.layers.61.mlp.experts.65.up_proj", "model.layers.61.mlp.experts.66.up_proj", "model.layers.61.mlp.experts.67.up_proj", "model.layers.61.mlp.experts.68.up_proj", "model.layers.61.mlp.experts.69.up_proj", "model.layers.61.mlp.experts.70.up_proj", "model.layers.61.mlp.experts.71.up_proj", "model.layers.61.mlp.experts.72.up_proj", "model.layers.61.mlp.experts.73.up_proj", "model.layers.61.mlp.experts.74.up_proj", "model.layers.61.mlp.experts.75.up_proj", "model.layers.61.mlp.experts.76.up_proj", "model.layers.61.mlp.experts.77.up_proj", "model.layers.61.mlp.experts.78.up_proj", "model.layers.61.mlp.experts.79.up_proj", "model.layers.61.mlp.experts.80.up_proj", "model.layers.61.mlp.experts.81.up_proj", "model.layers.61.mlp.experts.82.up_proj", "model.layers.61.mlp.experts.83.up_proj", "model.layers.61.mlp.experts.84.up_proj", "model.layers.61.mlp.experts.85.up_proj", "model.layers.61.mlp.experts.86.up_proj", "model.layers.61.mlp.experts.87.up_proj", "model.layers.61.mlp.experts.88.up_proj", "model.layers.61.mlp.experts.89.up_proj", "model.layers.61.mlp.experts.90.up_proj", "model.layers.61.mlp.experts.91.up_proj", "model.layers.61.mlp.experts.92.up_proj", "model.layers.61.mlp.experts.93.up_proj", "model.layers.61.mlp.experts.94.up_proj", "model.layers.61.mlp.experts.95.up_proj", "model.layers.61.mlp.experts.96.up_proj", "model.layers.61.mlp.experts.97.up_proj", "model.layers.61.mlp.experts.98.up_proj", "model.layers.61.mlp.experts.99.up_proj", "model.layers.61.mlp.experts.100.up_proj", "model.layers.61.mlp.experts.101.up_proj", "model.layers.61.mlp.experts.102.up_proj", "model.layers.61.mlp.experts.103.up_proj", "model.layers.61.mlp.experts.104.up_proj", "model.layers.61.mlp.experts.105.up_proj", "model.layers.61.mlp.experts.106.up_proj", "model.layers.61.mlp.experts.107.up_proj", "model.layers.61.mlp.experts.108.up_proj", "model.layers.61.mlp.experts.109.up_proj", "model.layers.61.mlp.experts.110.up_proj", "model.layers.61.mlp.experts.111.up_proj", "model.layers.61.mlp.experts.112.up_proj", "model.layers.61.mlp.experts.113.up_proj", "model.layers.61.mlp.experts.114.up_proj", "model.layers.61.mlp.experts.115.up_proj", "model.layers.61.mlp.experts.116.up_proj", "model.layers.61.mlp.experts.117.up_proj", "model.layers.61.mlp.experts.118.up_proj", "model.layers.61.mlp.experts.119.up_proj", "model.layers.61.mlp.experts.120.up_proj", "model.layers.61.mlp.experts.121.up_proj", "model.layers.61.mlp.experts.122.up_proj", "model.layers.61.mlp.experts.123.up_proj", "model.layers.61.mlp.experts.124.up_proj", "model.layers.61.mlp.experts.125.up_proj", "model.layers.61.mlp.experts.126.up_proj", "model.layers.61.mlp.experts.127.up_proj", "model.layers.61.mlp.experts.128.up_proj", "model.layers.61.mlp.experts.129.up_proj", "model.layers.61.mlp.experts.130.up_proj", "model.layers.61.mlp.experts.131.up_proj", "model.layers.61.mlp.experts.132.up_proj", "model.layers.61.mlp.experts.133.up_proj", "model.layers.61.mlp.experts.134.up_proj", "model.layers.61.mlp.experts.135.up_proj", "model.layers.61.mlp.experts.136.up_proj", "model.layers.61.mlp.experts.137.up_proj", "model.layers.61.mlp.experts.138.up_proj", "model.layers.61.mlp.experts.139.up_proj", "model.layers.61.mlp.experts.140.up_proj", "model.layers.61.mlp.experts.141.up_proj", "model.layers.61.mlp.experts.142.up_proj", "model.layers.61.mlp.experts.143.up_proj", "model.layers.61.mlp.experts.144.up_proj", "model.layers.61.mlp.experts.145.up_proj", "model.layers.61.mlp.experts.146.up_proj", "model.layers.61.mlp.experts.147.up_proj", "model.layers.61.mlp.experts.148.up_proj", "model.layers.61.mlp.experts.149.up_proj", "model.layers.61.mlp.experts.150.up_proj", "model.layers.61.mlp.experts.151.up_proj", "model.layers.61.mlp.experts.152.up_proj", "model.layers.61.mlp.experts.153.up_proj", "model.layers.61.mlp.experts.154.up_proj", "model.layers.61.mlp.experts.155.up_proj", "model.layers.61.mlp.experts.156.up_proj", "model.layers.61.mlp.experts.157.up_proj", "model.layers.61.mlp.experts.158.up_proj", "model.layers.61.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.492600469850047e-06, "dbits": 5033164800 } ] }, { "idx": 368, "layers": [ "model.layers.61.mlp.experts.0.down_proj", "model.layers.61.mlp.experts.1.down_proj", "model.layers.61.mlp.experts.2.down_proj", "model.layers.61.mlp.experts.3.down_proj", "model.layers.61.mlp.experts.4.down_proj", "model.layers.61.mlp.experts.5.down_proj", "model.layers.61.mlp.experts.6.down_proj", "model.layers.61.mlp.experts.7.down_proj", "model.layers.61.mlp.experts.8.down_proj", "model.layers.61.mlp.experts.9.down_proj", "model.layers.61.mlp.experts.10.down_proj", "model.layers.61.mlp.experts.11.down_proj", "model.layers.61.mlp.experts.12.down_proj", "model.layers.61.mlp.experts.13.down_proj", "model.layers.61.mlp.experts.14.down_proj", "model.layers.61.mlp.experts.15.down_proj", "model.layers.61.mlp.experts.16.down_proj", "model.layers.61.mlp.experts.17.down_proj", "model.layers.61.mlp.experts.18.down_proj", "model.layers.61.mlp.experts.19.down_proj", "model.layers.61.mlp.experts.20.down_proj", "model.layers.61.mlp.experts.21.down_proj", "model.layers.61.mlp.experts.22.down_proj", "model.layers.61.mlp.experts.23.down_proj", "model.layers.61.mlp.experts.24.down_proj", "model.layers.61.mlp.experts.25.down_proj", "model.layers.61.mlp.experts.26.down_proj", "model.layers.61.mlp.experts.27.down_proj", "model.layers.61.mlp.experts.28.down_proj", "model.layers.61.mlp.experts.29.down_proj", "model.layers.61.mlp.experts.30.down_proj", "model.layers.61.mlp.experts.31.down_proj", "model.layers.61.mlp.experts.32.down_proj", "model.layers.61.mlp.experts.33.down_proj", "model.layers.61.mlp.experts.34.down_proj", "model.layers.61.mlp.experts.35.down_proj", "model.layers.61.mlp.experts.36.down_proj", "model.layers.61.mlp.experts.37.down_proj", "model.layers.61.mlp.experts.38.down_proj", "model.layers.61.mlp.experts.39.down_proj", "model.layers.61.mlp.experts.40.down_proj", "model.layers.61.mlp.experts.41.down_proj", "model.layers.61.mlp.experts.42.down_proj", "model.layers.61.mlp.experts.43.down_proj", "model.layers.61.mlp.experts.44.down_proj", "model.layers.61.mlp.experts.45.down_proj", "model.layers.61.mlp.experts.46.down_proj", "model.layers.61.mlp.experts.47.down_proj", "model.layers.61.mlp.experts.48.down_proj", "model.layers.61.mlp.experts.49.down_proj", "model.layers.61.mlp.experts.50.down_proj", "model.layers.61.mlp.experts.51.down_proj", "model.layers.61.mlp.experts.52.down_proj", "model.layers.61.mlp.experts.53.down_proj", "model.layers.61.mlp.experts.54.down_proj", "model.layers.61.mlp.experts.55.down_proj", "model.layers.61.mlp.experts.56.down_proj", "model.layers.61.mlp.experts.57.down_proj", "model.layers.61.mlp.experts.58.down_proj", "model.layers.61.mlp.experts.59.down_proj", "model.layers.61.mlp.experts.60.down_proj", "model.layers.61.mlp.experts.61.down_proj", "model.layers.61.mlp.experts.62.down_proj", "model.layers.61.mlp.experts.63.down_proj", "model.layers.61.mlp.experts.64.down_proj", "model.layers.61.mlp.experts.65.down_proj", "model.layers.61.mlp.experts.66.down_proj", "model.layers.61.mlp.experts.67.down_proj", "model.layers.61.mlp.experts.68.down_proj", "model.layers.61.mlp.experts.69.down_proj", "model.layers.61.mlp.experts.70.down_proj", "model.layers.61.mlp.experts.71.down_proj", "model.layers.61.mlp.experts.72.down_proj", "model.layers.61.mlp.experts.73.down_proj", "model.layers.61.mlp.experts.74.down_proj", "model.layers.61.mlp.experts.75.down_proj", "model.layers.61.mlp.experts.76.down_proj", "model.layers.61.mlp.experts.77.down_proj", "model.layers.61.mlp.experts.78.down_proj", "model.layers.61.mlp.experts.79.down_proj", "model.layers.61.mlp.experts.80.down_proj", "model.layers.61.mlp.experts.81.down_proj", "model.layers.61.mlp.experts.82.down_proj", "model.layers.61.mlp.experts.83.down_proj", "model.layers.61.mlp.experts.84.down_proj", "model.layers.61.mlp.experts.85.down_proj", "model.layers.61.mlp.experts.86.down_proj", "model.layers.61.mlp.experts.87.down_proj", "model.layers.61.mlp.experts.88.down_proj", "model.layers.61.mlp.experts.89.down_proj", "model.layers.61.mlp.experts.90.down_proj", "model.layers.61.mlp.experts.91.down_proj", "model.layers.61.mlp.experts.92.down_proj", "model.layers.61.mlp.experts.93.down_proj", "model.layers.61.mlp.experts.94.down_proj", "model.layers.61.mlp.experts.95.down_proj", "model.layers.61.mlp.experts.96.down_proj", "model.layers.61.mlp.experts.97.down_proj", "model.layers.61.mlp.experts.98.down_proj", "model.layers.61.mlp.experts.99.down_proj", "model.layers.61.mlp.experts.100.down_proj", "model.layers.61.mlp.experts.101.down_proj", "model.layers.61.mlp.experts.102.down_proj", "model.layers.61.mlp.experts.103.down_proj", "model.layers.61.mlp.experts.104.down_proj", "model.layers.61.mlp.experts.105.down_proj", "model.layers.61.mlp.experts.106.down_proj", "model.layers.61.mlp.experts.107.down_proj", "model.layers.61.mlp.experts.108.down_proj", "model.layers.61.mlp.experts.109.down_proj", "model.layers.61.mlp.experts.110.down_proj", "model.layers.61.mlp.experts.111.down_proj", "model.layers.61.mlp.experts.112.down_proj", "model.layers.61.mlp.experts.113.down_proj", "model.layers.61.mlp.experts.114.down_proj", "model.layers.61.mlp.experts.115.down_proj", "model.layers.61.mlp.experts.116.down_proj", "model.layers.61.mlp.experts.117.down_proj", "model.layers.61.mlp.experts.118.down_proj", "model.layers.61.mlp.experts.119.down_proj", "model.layers.61.mlp.experts.120.down_proj", "model.layers.61.mlp.experts.121.down_proj", "model.layers.61.mlp.experts.122.down_proj", "model.layers.61.mlp.experts.123.down_proj", "model.layers.61.mlp.experts.124.down_proj", "model.layers.61.mlp.experts.125.down_proj", "model.layers.61.mlp.experts.126.down_proj", "model.layers.61.mlp.experts.127.down_proj", "model.layers.61.mlp.experts.128.down_proj", "model.layers.61.mlp.experts.129.down_proj", "model.layers.61.mlp.experts.130.down_proj", "model.layers.61.mlp.experts.131.down_proj", "model.layers.61.mlp.experts.132.down_proj", "model.layers.61.mlp.experts.133.down_proj", "model.layers.61.mlp.experts.134.down_proj", "model.layers.61.mlp.experts.135.down_proj", "model.layers.61.mlp.experts.136.down_proj", "model.layers.61.mlp.experts.137.down_proj", "model.layers.61.mlp.experts.138.down_proj", "model.layers.61.mlp.experts.139.down_proj", "model.layers.61.mlp.experts.140.down_proj", "model.layers.61.mlp.experts.141.down_proj", "model.layers.61.mlp.experts.142.down_proj", "model.layers.61.mlp.experts.143.down_proj", "model.layers.61.mlp.experts.144.down_proj", "model.layers.61.mlp.experts.145.down_proj", "model.layers.61.mlp.experts.146.down_proj", "model.layers.61.mlp.experts.147.down_proj", "model.layers.61.mlp.experts.148.down_proj", "model.layers.61.mlp.experts.149.down_proj", "model.layers.61.mlp.experts.150.down_proj", "model.layers.61.mlp.experts.151.down_proj", "model.layers.61.mlp.experts.152.down_proj", "model.layers.61.mlp.experts.153.down_proj", "model.layers.61.mlp.experts.154.down_proj", "model.layers.61.mlp.experts.155.down_proj", "model.layers.61.mlp.experts.156.down_proj", "model.layers.61.mlp.experts.157.down_proj", "model.layers.61.mlp.experts.158.down_proj", "model.layers.61.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.1270330762490847e-06, "dbits": 2516582400 } ] }, { "idx": 369, "layers": [ "model.layers.62.self_attn.q_proj" ], "candidates": [ { "dkld": 3.96046962123362e-06, "dbits": 125829120 } ] }, { "idx": 370, "layers": [ "model.layers.62.self_attn.k_proj", "model.layers.62.self_attn.v_proj" ], "candidates": [ { "dkld": 2.5770277716219425e-06, "dbits": 20971520 } ] }, { "idx": 371, "layers": [ "model.layers.62.self_attn.o_proj" ], "candidates": [ { "dkld": 3.021603333763778e-06, "dbits": 125829120 } ] }, { "idx": 372, "layers": [ "model.layers.62.mlp.shared_experts.gate_proj", "model.layers.62.mlp.shared_experts.up_proj", "model.layers.62.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.7346639651805596e-06, "dbits": 47185920 } ] }, { "idx": 373, "layers": [ "model.layers.62.mlp.experts.0.gate_proj", "model.layers.62.mlp.experts.1.gate_proj", "model.layers.62.mlp.experts.2.gate_proj", "model.layers.62.mlp.experts.3.gate_proj", "model.layers.62.mlp.experts.4.gate_proj", "model.layers.62.mlp.experts.5.gate_proj", "model.layers.62.mlp.experts.6.gate_proj", "model.layers.62.mlp.experts.7.gate_proj", "model.layers.62.mlp.experts.8.gate_proj", "model.layers.62.mlp.experts.9.gate_proj", "model.layers.62.mlp.experts.10.gate_proj", "model.layers.62.mlp.experts.11.gate_proj", "model.layers.62.mlp.experts.12.gate_proj", "model.layers.62.mlp.experts.13.gate_proj", "model.layers.62.mlp.experts.14.gate_proj", "model.layers.62.mlp.experts.15.gate_proj", "model.layers.62.mlp.experts.16.gate_proj", "model.layers.62.mlp.experts.17.gate_proj", "model.layers.62.mlp.experts.18.gate_proj", "model.layers.62.mlp.experts.19.gate_proj", "model.layers.62.mlp.experts.20.gate_proj", "model.layers.62.mlp.experts.21.gate_proj", "model.layers.62.mlp.experts.22.gate_proj", "model.layers.62.mlp.experts.23.gate_proj", "model.layers.62.mlp.experts.24.gate_proj", "model.layers.62.mlp.experts.25.gate_proj", "model.layers.62.mlp.experts.26.gate_proj", "model.layers.62.mlp.experts.27.gate_proj", "model.layers.62.mlp.experts.28.gate_proj", "model.layers.62.mlp.experts.29.gate_proj", "model.layers.62.mlp.experts.30.gate_proj", "model.layers.62.mlp.experts.31.gate_proj", "model.layers.62.mlp.experts.32.gate_proj", "model.layers.62.mlp.experts.33.gate_proj", "model.layers.62.mlp.experts.34.gate_proj", "model.layers.62.mlp.experts.35.gate_proj", "model.layers.62.mlp.experts.36.gate_proj", "model.layers.62.mlp.experts.37.gate_proj", "model.layers.62.mlp.experts.38.gate_proj", "model.layers.62.mlp.experts.39.gate_proj", "model.layers.62.mlp.experts.40.gate_proj", "model.layers.62.mlp.experts.41.gate_proj", "model.layers.62.mlp.experts.42.gate_proj", "model.layers.62.mlp.experts.43.gate_proj", "model.layers.62.mlp.experts.44.gate_proj", "model.layers.62.mlp.experts.45.gate_proj", "model.layers.62.mlp.experts.46.gate_proj", "model.layers.62.mlp.experts.47.gate_proj", "model.layers.62.mlp.experts.48.gate_proj", "model.layers.62.mlp.experts.49.gate_proj", "model.layers.62.mlp.experts.50.gate_proj", "model.layers.62.mlp.experts.51.gate_proj", "model.layers.62.mlp.experts.52.gate_proj", "model.layers.62.mlp.experts.53.gate_proj", "model.layers.62.mlp.experts.54.gate_proj", "model.layers.62.mlp.experts.55.gate_proj", "model.layers.62.mlp.experts.56.gate_proj", "model.layers.62.mlp.experts.57.gate_proj", "model.layers.62.mlp.experts.58.gate_proj", "model.layers.62.mlp.experts.59.gate_proj", "model.layers.62.mlp.experts.60.gate_proj", "model.layers.62.mlp.experts.61.gate_proj", "model.layers.62.mlp.experts.62.gate_proj", "model.layers.62.mlp.experts.63.gate_proj", "model.layers.62.mlp.experts.64.gate_proj", "model.layers.62.mlp.experts.65.gate_proj", "model.layers.62.mlp.experts.66.gate_proj", "model.layers.62.mlp.experts.67.gate_proj", "model.layers.62.mlp.experts.68.gate_proj", "model.layers.62.mlp.experts.69.gate_proj", "model.layers.62.mlp.experts.70.gate_proj", "model.layers.62.mlp.experts.71.gate_proj", "model.layers.62.mlp.experts.72.gate_proj", "model.layers.62.mlp.experts.73.gate_proj", "model.layers.62.mlp.experts.74.gate_proj", "model.layers.62.mlp.experts.75.gate_proj", "model.layers.62.mlp.experts.76.gate_proj", "model.layers.62.mlp.experts.77.gate_proj", "model.layers.62.mlp.experts.78.gate_proj", "model.layers.62.mlp.experts.79.gate_proj", "model.layers.62.mlp.experts.80.gate_proj", "model.layers.62.mlp.experts.81.gate_proj", "model.layers.62.mlp.experts.82.gate_proj", "model.layers.62.mlp.experts.83.gate_proj", "model.layers.62.mlp.experts.84.gate_proj", "model.layers.62.mlp.experts.85.gate_proj", "model.layers.62.mlp.experts.86.gate_proj", "model.layers.62.mlp.experts.87.gate_proj", "model.layers.62.mlp.experts.88.gate_proj", "model.layers.62.mlp.experts.89.gate_proj", "model.layers.62.mlp.experts.90.gate_proj", "model.layers.62.mlp.experts.91.gate_proj", "model.layers.62.mlp.experts.92.gate_proj", "model.layers.62.mlp.experts.93.gate_proj", "model.layers.62.mlp.experts.94.gate_proj", "model.layers.62.mlp.experts.95.gate_proj", "model.layers.62.mlp.experts.96.gate_proj", "model.layers.62.mlp.experts.97.gate_proj", "model.layers.62.mlp.experts.98.gate_proj", "model.layers.62.mlp.experts.99.gate_proj", "model.layers.62.mlp.experts.100.gate_proj", "model.layers.62.mlp.experts.101.gate_proj", "model.layers.62.mlp.experts.102.gate_proj", "model.layers.62.mlp.experts.103.gate_proj", "model.layers.62.mlp.experts.104.gate_proj", "model.layers.62.mlp.experts.105.gate_proj", "model.layers.62.mlp.experts.106.gate_proj", "model.layers.62.mlp.experts.107.gate_proj", "model.layers.62.mlp.experts.108.gate_proj", "model.layers.62.mlp.experts.109.gate_proj", "model.layers.62.mlp.experts.110.gate_proj", "model.layers.62.mlp.experts.111.gate_proj", "model.layers.62.mlp.experts.112.gate_proj", "model.layers.62.mlp.experts.113.gate_proj", "model.layers.62.mlp.experts.114.gate_proj", "model.layers.62.mlp.experts.115.gate_proj", "model.layers.62.mlp.experts.116.gate_proj", "model.layers.62.mlp.experts.117.gate_proj", "model.layers.62.mlp.experts.118.gate_proj", "model.layers.62.mlp.experts.119.gate_proj", "model.layers.62.mlp.experts.120.gate_proj", "model.layers.62.mlp.experts.121.gate_proj", "model.layers.62.mlp.experts.122.gate_proj", "model.layers.62.mlp.experts.123.gate_proj", "model.layers.62.mlp.experts.124.gate_proj", "model.layers.62.mlp.experts.125.gate_proj", "model.layers.62.mlp.experts.126.gate_proj", "model.layers.62.mlp.experts.127.gate_proj", "model.layers.62.mlp.experts.128.gate_proj", "model.layers.62.mlp.experts.129.gate_proj", "model.layers.62.mlp.experts.130.gate_proj", "model.layers.62.mlp.experts.131.gate_proj", "model.layers.62.mlp.experts.132.gate_proj", "model.layers.62.mlp.experts.133.gate_proj", "model.layers.62.mlp.experts.134.gate_proj", "model.layers.62.mlp.experts.135.gate_proj", "model.layers.62.mlp.experts.136.gate_proj", "model.layers.62.mlp.experts.137.gate_proj", "model.layers.62.mlp.experts.138.gate_proj", "model.layers.62.mlp.experts.139.gate_proj", "model.layers.62.mlp.experts.140.gate_proj", "model.layers.62.mlp.experts.141.gate_proj", "model.layers.62.mlp.experts.142.gate_proj", "model.layers.62.mlp.experts.143.gate_proj", "model.layers.62.mlp.experts.144.gate_proj", "model.layers.62.mlp.experts.145.gate_proj", "model.layers.62.mlp.experts.146.gate_proj", "model.layers.62.mlp.experts.147.gate_proj", "model.layers.62.mlp.experts.148.gate_proj", "model.layers.62.mlp.experts.149.gate_proj", "model.layers.62.mlp.experts.150.gate_proj", "model.layers.62.mlp.experts.151.gate_proj", "model.layers.62.mlp.experts.152.gate_proj", "model.layers.62.mlp.experts.153.gate_proj", "model.layers.62.mlp.experts.154.gate_proj", "model.layers.62.mlp.experts.155.gate_proj", "model.layers.62.mlp.experts.156.gate_proj", "model.layers.62.mlp.experts.157.gate_proj", "model.layers.62.mlp.experts.158.gate_proj", "model.layers.62.mlp.experts.159.gate_proj", "model.layers.62.mlp.experts.0.up_proj", "model.layers.62.mlp.experts.1.up_proj", "model.layers.62.mlp.experts.2.up_proj", "model.layers.62.mlp.experts.3.up_proj", "model.layers.62.mlp.experts.4.up_proj", "model.layers.62.mlp.experts.5.up_proj", "model.layers.62.mlp.experts.6.up_proj", "model.layers.62.mlp.experts.7.up_proj", "model.layers.62.mlp.experts.8.up_proj", "model.layers.62.mlp.experts.9.up_proj", "model.layers.62.mlp.experts.10.up_proj", "model.layers.62.mlp.experts.11.up_proj", "model.layers.62.mlp.experts.12.up_proj", "model.layers.62.mlp.experts.13.up_proj", "model.layers.62.mlp.experts.14.up_proj", "model.layers.62.mlp.experts.15.up_proj", "model.layers.62.mlp.experts.16.up_proj", "model.layers.62.mlp.experts.17.up_proj", "model.layers.62.mlp.experts.18.up_proj", "model.layers.62.mlp.experts.19.up_proj", "model.layers.62.mlp.experts.20.up_proj", "model.layers.62.mlp.experts.21.up_proj", "model.layers.62.mlp.experts.22.up_proj", "model.layers.62.mlp.experts.23.up_proj", "model.layers.62.mlp.experts.24.up_proj", "model.layers.62.mlp.experts.25.up_proj", "model.layers.62.mlp.experts.26.up_proj", "model.layers.62.mlp.experts.27.up_proj", "model.layers.62.mlp.experts.28.up_proj", "model.layers.62.mlp.experts.29.up_proj", "model.layers.62.mlp.experts.30.up_proj", "model.layers.62.mlp.experts.31.up_proj", "model.layers.62.mlp.experts.32.up_proj", "model.layers.62.mlp.experts.33.up_proj", "model.layers.62.mlp.experts.34.up_proj", "model.layers.62.mlp.experts.35.up_proj", "model.layers.62.mlp.experts.36.up_proj", "model.layers.62.mlp.experts.37.up_proj", "model.layers.62.mlp.experts.38.up_proj", "model.layers.62.mlp.experts.39.up_proj", "model.layers.62.mlp.experts.40.up_proj", "model.layers.62.mlp.experts.41.up_proj", "model.layers.62.mlp.experts.42.up_proj", "model.layers.62.mlp.experts.43.up_proj", "model.layers.62.mlp.experts.44.up_proj", "model.layers.62.mlp.experts.45.up_proj", "model.layers.62.mlp.experts.46.up_proj", "model.layers.62.mlp.experts.47.up_proj", "model.layers.62.mlp.experts.48.up_proj", "model.layers.62.mlp.experts.49.up_proj", "model.layers.62.mlp.experts.50.up_proj", "model.layers.62.mlp.experts.51.up_proj", "model.layers.62.mlp.experts.52.up_proj", "model.layers.62.mlp.experts.53.up_proj", "model.layers.62.mlp.experts.54.up_proj", "model.layers.62.mlp.experts.55.up_proj", "model.layers.62.mlp.experts.56.up_proj", "model.layers.62.mlp.experts.57.up_proj", "model.layers.62.mlp.experts.58.up_proj", "model.layers.62.mlp.experts.59.up_proj", "model.layers.62.mlp.experts.60.up_proj", "model.layers.62.mlp.experts.61.up_proj", "model.layers.62.mlp.experts.62.up_proj", "model.layers.62.mlp.experts.63.up_proj", "model.layers.62.mlp.experts.64.up_proj", "model.layers.62.mlp.experts.65.up_proj", "model.layers.62.mlp.experts.66.up_proj", "model.layers.62.mlp.experts.67.up_proj", "model.layers.62.mlp.experts.68.up_proj", "model.layers.62.mlp.experts.69.up_proj", "model.layers.62.mlp.experts.70.up_proj", "model.layers.62.mlp.experts.71.up_proj", "model.layers.62.mlp.experts.72.up_proj", "model.layers.62.mlp.experts.73.up_proj", "model.layers.62.mlp.experts.74.up_proj", "model.layers.62.mlp.experts.75.up_proj", "model.layers.62.mlp.experts.76.up_proj", "model.layers.62.mlp.experts.77.up_proj", "model.layers.62.mlp.experts.78.up_proj", "model.layers.62.mlp.experts.79.up_proj", "model.layers.62.mlp.experts.80.up_proj", "model.layers.62.mlp.experts.81.up_proj", "model.layers.62.mlp.experts.82.up_proj", "model.layers.62.mlp.experts.83.up_proj", "model.layers.62.mlp.experts.84.up_proj", "model.layers.62.mlp.experts.85.up_proj", "model.layers.62.mlp.experts.86.up_proj", "model.layers.62.mlp.experts.87.up_proj", "model.layers.62.mlp.experts.88.up_proj", "model.layers.62.mlp.experts.89.up_proj", "model.layers.62.mlp.experts.90.up_proj", "model.layers.62.mlp.experts.91.up_proj", "model.layers.62.mlp.experts.92.up_proj", "model.layers.62.mlp.experts.93.up_proj", "model.layers.62.mlp.experts.94.up_proj", "model.layers.62.mlp.experts.95.up_proj", "model.layers.62.mlp.experts.96.up_proj", "model.layers.62.mlp.experts.97.up_proj", "model.layers.62.mlp.experts.98.up_proj", "model.layers.62.mlp.experts.99.up_proj", "model.layers.62.mlp.experts.100.up_proj", "model.layers.62.mlp.experts.101.up_proj", "model.layers.62.mlp.experts.102.up_proj", "model.layers.62.mlp.experts.103.up_proj", "model.layers.62.mlp.experts.104.up_proj", "model.layers.62.mlp.experts.105.up_proj", "model.layers.62.mlp.experts.106.up_proj", "model.layers.62.mlp.experts.107.up_proj", "model.layers.62.mlp.experts.108.up_proj", "model.layers.62.mlp.experts.109.up_proj", "model.layers.62.mlp.experts.110.up_proj", "model.layers.62.mlp.experts.111.up_proj", "model.layers.62.mlp.experts.112.up_proj", "model.layers.62.mlp.experts.113.up_proj", "model.layers.62.mlp.experts.114.up_proj", "model.layers.62.mlp.experts.115.up_proj", "model.layers.62.mlp.experts.116.up_proj", "model.layers.62.mlp.experts.117.up_proj", "model.layers.62.mlp.experts.118.up_proj", "model.layers.62.mlp.experts.119.up_proj", "model.layers.62.mlp.experts.120.up_proj", "model.layers.62.mlp.experts.121.up_proj", "model.layers.62.mlp.experts.122.up_proj", "model.layers.62.mlp.experts.123.up_proj", "model.layers.62.mlp.experts.124.up_proj", "model.layers.62.mlp.experts.125.up_proj", "model.layers.62.mlp.experts.126.up_proj", "model.layers.62.mlp.experts.127.up_proj", "model.layers.62.mlp.experts.128.up_proj", "model.layers.62.mlp.experts.129.up_proj", "model.layers.62.mlp.experts.130.up_proj", "model.layers.62.mlp.experts.131.up_proj", "model.layers.62.mlp.experts.132.up_proj", "model.layers.62.mlp.experts.133.up_proj", "model.layers.62.mlp.experts.134.up_proj", "model.layers.62.mlp.experts.135.up_proj", "model.layers.62.mlp.experts.136.up_proj", "model.layers.62.mlp.experts.137.up_proj", "model.layers.62.mlp.experts.138.up_proj", "model.layers.62.mlp.experts.139.up_proj", "model.layers.62.mlp.experts.140.up_proj", "model.layers.62.mlp.experts.141.up_proj", "model.layers.62.mlp.experts.142.up_proj", "model.layers.62.mlp.experts.143.up_proj", "model.layers.62.mlp.experts.144.up_proj", "model.layers.62.mlp.experts.145.up_proj", "model.layers.62.mlp.experts.146.up_proj", "model.layers.62.mlp.experts.147.up_proj", "model.layers.62.mlp.experts.148.up_proj", "model.layers.62.mlp.experts.149.up_proj", "model.layers.62.mlp.experts.150.up_proj", "model.layers.62.mlp.experts.151.up_proj", "model.layers.62.mlp.experts.152.up_proj", "model.layers.62.mlp.experts.153.up_proj", "model.layers.62.mlp.experts.154.up_proj", "model.layers.62.mlp.experts.155.up_proj", "model.layers.62.mlp.experts.156.up_proj", "model.layers.62.mlp.experts.157.up_proj", "model.layers.62.mlp.experts.158.up_proj", "model.layers.62.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.756310954690023e-06, "dbits": 5033164800 } ] }, { "idx": 374, "layers": [ "model.layers.62.mlp.experts.0.down_proj", "model.layers.62.mlp.experts.1.down_proj", "model.layers.62.mlp.experts.2.down_proj", "model.layers.62.mlp.experts.3.down_proj", "model.layers.62.mlp.experts.4.down_proj", "model.layers.62.mlp.experts.5.down_proj", "model.layers.62.mlp.experts.6.down_proj", "model.layers.62.mlp.experts.7.down_proj", "model.layers.62.mlp.experts.8.down_proj", "model.layers.62.mlp.experts.9.down_proj", "model.layers.62.mlp.experts.10.down_proj", "model.layers.62.mlp.experts.11.down_proj", "model.layers.62.mlp.experts.12.down_proj", "model.layers.62.mlp.experts.13.down_proj", "model.layers.62.mlp.experts.14.down_proj", "model.layers.62.mlp.experts.15.down_proj", "model.layers.62.mlp.experts.16.down_proj", "model.layers.62.mlp.experts.17.down_proj", "model.layers.62.mlp.experts.18.down_proj", "model.layers.62.mlp.experts.19.down_proj", "model.layers.62.mlp.experts.20.down_proj", "model.layers.62.mlp.experts.21.down_proj", "model.layers.62.mlp.experts.22.down_proj", "model.layers.62.mlp.experts.23.down_proj", "model.layers.62.mlp.experts.24.down_proj", "model.layers.62.mlp.experts.25.down_proj", "model.layers.62.mlp.experts.26.down_proj", "model.layers.62.mlp.experts.27.down_proj", "model.layers.62.mlp.experts.28.down_proj", "model.layers.62.mlp.experts.29.down_proj", "model.layers.62.mlp.experts.30.down_proj", "model.layers.62.mlp.experts.31.down_proj", "model.layers.62.mlp.experts.32.down_proj", "model.layers.62.mlp.experts.33.down_proj", "model.layers.62.mlp.experts.34.down_proj", "model.layers.62.mlp.experts.35.down_proj", "model.layers.62.mlp.experts.36.down_proj", "model.layers.62.mlp.experts.37.down_proj", "model.layers.62.mlp.experts.38.down_proj", "model.layers.62.mlp.experts.39.down_proj", "model.layers.62.mlp.experts.40.down_proj", "model.layers.62.mlp.experts.41.down_proj", "model.layers.62.mlp.experts.42.down_proj", "model.layers.62.mlp.experts.43.down_proj", "model.layers.62.mlp.experts.44.down_proj", "model.layers.62.mlp.experts.45.down_proj", "model.layers.62.mlp.experts.46.down_proj", "model.layers.62.mlp.experts.47.down_proj", "model.layers.62.mlp.experts.48.down_proj", "model.layers.62.mlp.experts.49.down_proj", "model.layers.62.mlp.experts.50.down_proj", "model.layers.62.mlp.experts.51.down_proj", "model.layers.62.mlp.experts.52.down_proj", "model.layers.62.mlp.experts.53.down_proj", "model.layers.62.mlp.experts.54.down_proj", "model.layers.62.mlp.experts.55.down_proj", "model.layers.62.mlp.experts.56.down_proj", "model.layers.62.mlp.experts.57.down_proj", "model.layers.62.mlp.experts.58.down_proj", "model.layers.62.mlp.experts.59.down_proj", "model.layers.62.mlp.experts.60.down_proj", "model.layers.62.mlp.experts.61.down_proj", "model.layers.62.mlp.experts.62.down_proj", "model.layers.62.mlp.experts.63.down_proj", "model.layers.62.mlp.experts.64.down_proj", "model.layers.62.mlp.experts.65.down_proj", "model.layers.62.mlp.experts.66.down_proj", "model.layers.62.mlp.experts.67.down_proj", "model.layers.62.mlp.experts.68.down_proj", "model.layers.62.mlp.experts.69.down_proj", "model.layers.62.mlp.experts.70.down_proj", "model.layers.62.mlp.experts.71.down_proj", "model.layers.62.mlp.experts.72.down_proj", "model.layers.62.mlp.experts.73.down_proj", "model.layers.62.mlp.experts.74.down_proj", "model.layers.62.mlp.experts.75.down_proj", "model.layers.62.mlp.experts.76.down_proj", "model.layers.62.mlp.experts.77.down_proj", "model.layers.62.mlp.experts.78.down_proj", "model.layers.62.mlp.experts.79.down_proj", "model.layers.62.mlp.experts.80.down_proj", "model.layers.62.mlp.experts.81.down_proj", "model.layers.62.mlp.experts.82.down_proj", "model.layers.62.mlp.experts.83.down_proj", "model.layers.62.mlp.experts.84.down_proj", "model.layers.62.mlp.experts.85.down_proj", "model.layers.62.mlp.experts.86.down_proj", "model.layers.62.mlp.experts.87.down_proj", "model.layers.62.mlp.experts.88.down_proj", "model.layers.62.mlp.experts.89.down_proj", "model.layers.62.mlp.experts.90.down_proj", "model.layers.62.mlp.experts.91.down_proj", "model.layers.62.mlp.experts.92.down_proj", "model.layers.62.mlp.experts.93.down_proj", "model.layers.62.mlp.experts.94.down_proj", "model.layers.62.mlp.experts.95.down_proj", "model.layers.62.mlp.experts.96.down_proj", "model.layers.62.mlp.experts.97.down_proj", "model.layers.62.mlp.experts.98.down_proj", "model.layers.62.mlp.experts.99.down_proj", "model.layers.62.mlp.experts.100.down_proj", "model.layers.62.mlp.experts.101.down_proj", "model.layers.62.mlp.experts.102.down_proj", "model.layers.62.mlp.experts.103.down_proj", "model.layers.62.mlp.experts.104.down_proj", "model.layers.62.mlp.experts.105.down_proj", "model.layers.62.mlp.experts.106.down_proj", "model.layers.62.mlp.experts.107.down_proj", "model.layers.62.mlp.experts.108.down_proj", "model.layers.62.mlp.experts.109.down_proj", "model.layers.62.mlp.experts.110.down_proj", "model.layers.62.mlp.experts.111.down_proj", "model.layers.62.mlp.experts.112.down_proj", "model.layers.62.mlp.experts.113.down_proj", "model.layers.62.mlp.experts.114.down_proj", "model.layers.62.mlp.experts.115.down_proj", "model.layers.62.mlp.experts.116.down_proj", "model.layers.62.mlp.experts.117.down_proj", "model.layers.62.mlp.experts.118.down_proj", "model.layers.62.mlp.experts.119.down_proj", "model.layers.62.mlp.experts.120.down_proj", "model.layers.62.mlp.experts.121.down_proj", "model.layers.62.mlp.experts.122.down_proj", "model.layers.62.mlp.experts.123.down_proj", "model.layers.62.mlp.experts.124.down_proj", "model.layers.62.mlp.experts.125.down_proj", "model.layers.62.mlp.experts.126.down_proj", "model.layers.62.mlp.experts.127.down_proj", "model.layers.62.mlp.experts.128.down_proj", "model.layers.62.mlp.experts.129.down_proj", "model.layers.62.mlp.experts.130.down_proj", "model.layers.62.mlp.experts.131.down_proj", "model.layers.62.mlp.experts.132.down_proj", "model.layers.62.mlp.experts.133.down_proj", "model.layers.62.mlp.experts.134.down_proj", "model.layers.62.mlp.experts.135.down_proj", "model.layers.62.mlp.experts.136.down_proj", "model.layers.62.mlp.experts.137.down_proj", "model.layers.62.mlp.experts.138.down_proj", "model.layers.62.mlp.experts.139.down_proj", "model.layers.62.mlp.experts.140.down_proj", "model.layers.62.mlp.experts.141.down_proj", "model.layers.62.mlp.experts.142.down_proj", "model.layers.62.mlp.experts.143.down_proj", "model.layers.62.mlp.experts.144.down_proj", "model.layers.62.mlp.experts.145.down_proj", "model.layers.62.mlp.experts.146.down_proj", "model.layers.62.mlp.experts.147.down_proj", "model.layers.62.mlp.experts.148.down_proj", "model.layers.62.mlp.experts.149.down_proj", "model.layers.62.mlp.experts.150.down_proj", "model.layers.62.mlp.experts.151.down_proj", "model.layers.62.mlp.experts.152.down_proj", "model.layers.62.mlp.experts.153.down_proj", "model.layers.62.mlp.experts.154.down_proj", "model.layers.62.mlp.experts.155.down_proj", "model.layers.62.mlp.experts.156.down_proj", "model.layers.62.mlp.experts.157.down_proj", "model.layers.62.mlp.experts.158.down_proj", "model.layers.62.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.484176027588503e-06, "dbits": 2516582400 } ] }, { "idx": 375, "layers": [ "model.layers.63.self_attn.q_proj" ], "candidates": [ { "dkld": 1.3597105862573006e-06, "dbits": 125829120 } ] }, { "idx": 376, "layers": [ "model.layers.63.self_attn.k_proj", "model.layers.63.self_attn.v_proj" ], "candidates": [ { "dkld": -3.0893716029822826e-08, "dbits": 20971520 } ] }, { "idx": 377, "layers": [ "model.layers.63.self_attn.o_proj" ], "candidates": [ { "dkld": 2.9918999643995924e-06, "dbits": 125829120 } ] }, { "idx": 378, "layers": [ "model.layers.63.mlp.shared_experts.gate_proj", "model.layers.63.mlp.shared_experts.up_proj", "model.layers.63.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.766986679285743e-06, "dbits": 47185920 } ] }, { "idx": 379, "layers": [ "model.layers.63.mlp.experts.0.gate_proj", "model.layers.63.mlp.experts.1.gate_proj", "model.layers.63.mlp.experts.2.gate_proj", "model.layers.63.mlp.experts.3.gate_proj", "model.layers.63.mlp.experts.4.gate_proj", "model.layers.63.mlp.experts.5.gate_proj", "model.layers.63.mlp.experts.6.gate_proj", "model.layers.63.mlp.experts.7.gate_proj", "model.layers.63.mlp.experts.8.gate_proj", "model.layers.63.mlp.experts.9.gate_proj", "model.layers.63.mlp.experts.10.gate_proj", "model.layers.63.mlp.experts.11.gate_proj", "model.layers.63.mlp.experts.12.gate_proj", "model.layers.63.mlp.experts.13.gate_proj", "model.layers.63.mlp.experts.14.gate_proj", "model.layers.63.mlp.experts.15.gate_proj", "model.layers.63.mlp.experts.16.gate_proj", "model.layers.63.mlp.experts.17.gate_proj", "model.layers.63.mlp.experts.18.gate_proj", "model.layers.63.mlp.experts.19.gate_proj", "model.layers.63.mlp.experts.20.gate_proj", "model.layers.63.mlp.experts.21.gate_proj", "model.layers.63.mlp.experts.22.gate_proj", "model.layers.63.mlp.experts.23.gate_proj", "model.layers.63.mlp.experts.24.gate_proj", "model.layers.63.mlp.experts.25.gate_proj", "model.layers.63.mlp.experts.26.gate_proj", "model.layers.63.mlp.experts.27.gate_proj", "model.layers.63.mlp.experts.28.gate_proj", "model.layers.63.mlp.experts.29.gate_proj", "model.layers.63.mlp.experts.30.gate_proj", "model.layers.63.mlp.experts.31.gate_proj", "model.layers.63.mlp.experts.32.gate_proj", "model.layers.63.mlp.experts.33.gate_proj", "model.layers.63.mlp.experts.34.gate_proj", "model.layers.63.mlp.experts.35.gate_proj", "model.layers.63.mlp.experts.36.gate_proj", "model.layers.63.mlp.experts.37.gate_proj", "model.layers.63.mlp.experts.38.gate_proj", "model.layers.63.mlp.experts.39.gate_proj", "model.layers.63.mlp.experts.40.gate_proj", "model.layers.63.mlp.experts.41.gate_proj", "model.layers.63.mlp.experts.42.gate_proj", "model.layers.63.mlp.experts.43.gate_proj", "model.layers.63.mlp.experts.44.gate_proj", "model.layers.63.mlp.experts.45.gate_proj", "model.layers.63.mlp.experts.46.gate_proj", "model.layers.63.mlp.experts.47.gate_proj", "model.layers.63.mlp.experts.48.gate_proj", "model.layers.63.mlp.experts.49.gate_proj", "model.layers.63.mlp.experts.50.gate_proj", "model.layers.63.mlp.experts.51.gate_proj", "model.layers.63.mlp.experts.52.gate_proj", "model.layers.63.mlp.experts.53.gate_proj", "model.layers.63.mlp.experts.54.gate_proj", "model.layers.63.mlp.experts.55.gate_proj", "model.layers.63.mlp.experts.56.gate_proj", "model.layers.63.mlp.experts.57.gate_proj", "model.layers.63.mlp.experts.58.gate_proj", "model.layers.63.mlp.experts.59.gate_proj", "model.layers.63.mlp.experts.60.gate_proj", "model.layers.63.mlp.experts.61.gate_proj", "model.layers.63.mlp.experts.62.gate_proj", "model.layers.63.mlp.experts.63.gate_proj", "model.layers.63.mlp.experts.64.gate_proj", "model.layers.63.mlp.experts.65.gate_proj", "model.layers.63.mlp.experts.66.gate_proj", "model.layers.63.mlp.experts.67.gate_proj", "model.layers.63.mlp.experts.68.gate_proj", "model.layers.63.mlp.experts.69.gate_proj", "model.layers.63.mlp.experts.70.gate_proj", "model.layers.63.mlp.experts.71.gate_proj", "model.layers.63.mlp.experts.72.gate_proj", "model.layers.63.mlp.experts.73.gate_proj", "model.layers.63.mlp.experts.74.gate_proj", "model.layers.63.mlp.experts.75.gate_proj", "model.layers.63.mlp.experts.76.gate_proj", "model.layers.63.mlp.experts.77.gate_proj", "model.layers.63.mlp.experts.78.gate_proj", "model.layers.63.mlp.experts.79.gate_proj", "model.layers.63.mlp.experts.80.gate_proj", "model.layers.63.mlp.experts.81.gate_proj", "model.layers.63.mlp.experts.82.gate_proj", "model.layers.63.mlp.experts.83.gate_proj", "model.layers.63.mlp.experts.84.gate_proj", "model.layers.63.mlp.experts.85.gate_proj", "model.layers.63.mlp.experts.86.gate_proj", "model.layers.63.mlp.experts.87.gate_proj", "model.layers.63.mlp.experts.88.gate_proj", "model.layers.63.mlp.experts.89.gate_proj", "model.layers.63.mlp.experts.90.gate_proj", "model.layers.63.mlp.experts.91.gate_proj", "model.layers.63.mlp.experts.92.gate_proj", "model.layers.63.mlp.experts.93.gate_proj", "model.layers.63.mlp.experts.94.gate_proj", "model.layers.63.mlp.experts.95.gate_proj", "model.layers.63.mlp.experts.96.gate_proj", "model.layers.63.mlp.experts.97.gate_proj", "model.layers.63.mlp.experts.98.gate_proj", "model.layers.63.mlp.experts.99.gate_proj", "model.layers.63.mlp.experts.100.gate_proj", "model.layers.63.mlp.experts.101.gate_proj", "model.layers.63.mlp.experts.102.gate_proj", "model.layers.63.mlp.experts.103.gate_proj", "model.layers.63.mlp.experts.104.gate_proj", "model.layers.63.mlp.experts.105.gate_proj", "model.layers.63.mlp.experts.106.gate_proj", "model.layers.63.mlp.experts.107.gate_proj", "model.layers.63.mlp.experts.108.gate_proj", "model.layers.63.mlp.experts.109.gate_proj", "model.layers.63.mlp.experts.110.gate_proj", "model.layers.63.mlp.experts.111.gate_proj", "model.layers.63.mlp.experts.112.gate_proj", "model.layers.63.mlp.experts.113.gate_proj", "model.layers.63.mlp.experts.114.gate_proj", "model.layers.63.mlp.experts.115.gate_proj", "model.layers.63.mlp.experts.116.gate_proj", "model.layers.63.mlp.experts.117.gate_proj", "model.layers.63.mlp.experts.118.gate_proj", "model.layers.63.mlp.experts.119.gate_proj", "model.layers.63.mlp.experts.120.gate_proj", "model.layers.63.mlp.experts.121.gate_proj", "model.layers.63.mlp.experts.122.gate_proj", "model.layers.63.mlp.experts.123.gate_proj", "model.layers.63.mlp.experts.124.gate_proj", "model.layers.63.mlp.experts.125.gate_proj", "model.layers.63.mlp.experts.126.gate_proj", "model.layers.63.mlp.experts.127.gate_proj", "model.layers.63.mlp.experts.128.gate_proj", "model.layers.63.mlp.experts.129.gate_proj", "model.layers.63.mlp.experts.130.gate_proj", "model.layers.63.mlp.experts.131.gate_proj", "model.layers.63.mlp.experts.132.gate_proj", "model.layers.63.mlp.experts.133.gate_proj", "model.layers.63.mlp.experts.134.gate_proj", "model.layers.63.mlp.experts.135.gate_proj", "model.layers.63.mlp.experts.136.gate_proj", "model.layers.63.mlp.experts.137.gate_proj", "model.layers.63.mlp.experts.138.gate_proj", "model.layers.63.mlp.experts.139.gate_proj", "model.layers.63.mlp.experts.140.gate_proj", "model.layers.63.mlp.experts.141.gate_proj", "model.layers.63.mlp.experts.142.gate_proj", "model.layers.63.mlp.experts.143.gate_proj", "model.layers.63.mlp.experts.144.gate_proj", "model.layers.63.mlp.experts.145.gate_proj", "model.layers.63.mlp.experts.146.gate_proj", "model.layers.63.mlp.experts.147.gate_proj", "model.layers.63.mlp.experts.148.gate_proj", "model.layers.63.mlp.experts.149.gate_proj", "model.layers.63.mlp.experts.150.gate_proj", "model.layers.63.mlp.experts.151.gate_proj", "model.layers.63.mlp.experts.152.gate_proj", "model.layers.63.mlp.experts.153.gate_proj", "model.layers.63.mlp.experts.154.gate_proj", "model.layers.63.mlp.experts.155.gate_proj", "model.layers.63.mlp.experts.156.gate_proj", "model.layers.63.mlp.experts.157.gate_proj", "model.layers.63.mlp.experts.158.gate_proj", "model.layers.63.mlp.experts.159.gate_proj", "model.layers.63.mlp.experts.0.up_proj", "model.layers.63.mlp.experts.1.up_proj", "model.layers.63.mlp.experts.2.up_proj", "model.layers.63.mlp.experts.3.up_proj", "model.layers.63.mlp.experts.4.up_proj", "model.layers.63.mlp.experts.5.up_proj", "model.layers.63.mlp.experts.6.up_proj", "model.layers.63.mlp.experts.7.up_proj", "model.layers.63.mlp.experts.8.up_proj", "model.layers.63.mlp.experts.9.up_proj", "model.layers.63.mlp.experts.10.up_proj", "model.layers.63.mlp.experts.11.up_proj", "model.layers.63.mlp.experts.12.up_proj", "model.layers.63.mlp.experts.13.up_proj", "model.layers.63.mlp.experts.14.up_proj", "model.layers.63.mlp.experts.15.up_proj", "model.layers.63.mlp.experts.16.up_proj", "model.layers.63.mlp.experts.17.up_proj", "model.layers.63.mlp.experts.18.up_proj", "model.layers.63.mlp.experts.19.up_proj", "model.layers.63.mlp.experts.20.up_proj", "model.layers.63.mlp.experts.21.up_proj", "model.layers.63.mlp.experts.22.up_proj", "model.layers.63.mlp.experts.23.up_proj", "model.layers.63.mlp.experts.24.up_proj", "model.layers.63.mlp.experts.25.up_proj", "model.layers.63.mlp.experts.26.up_proj", "model.layers.63.mlp.experts.27.up_proj", "model.layers.63.mlp.experts.28.up_proj", "model.layers.63.mlp.experts.29.up_proj", "model.layers.63.mlp.experts.30.up_proj", "model.layers.63.mlp.experts.31.up_proj", "model.layers.63.mlp.experts.32.up_proj", "model.layers.63.mlp.experts.33.up_proj", "model.layers.63.mlp.experts.34.up_proj", "model.layers.63.mlp.experts.35.up_proj", "model.layers.63.mlp.experts.36.up_proj", "model.layers.63.mlp.experts.37.up_proj", "model.layers.63.mlp.experts.38.up_proj", "model.layers.63.mlp.experts.39.up_proj", "model.layers.63.mlp.experts.40.up_proj", "model.layers.63.mlp.experts.41.up_proj", "model.layers.63.mlp.experts.42.up_proj", "model.layers.63.mlp.experts.43.up_proj", "model.layers.63.mlp.experts.44.up_proj", "model.layers.63.mlp.experts.45.up_proj", "model.layers.63.mlp.experts.46.up_proj", "model.layers.63.mlp.experts.47.up_proj", "model.layers.63.mlp.experts.48.up_proj", "model.layers.63.mlp.experts.49.up_proj", "model.layers.63.mlp.experts.50.up_proj", "model.layers.63.mlp.experts.51.up_proj", "model.layers.63.mlp.experts.52.up_proj", "model.layers.63.mlp.experts.53.up_proj", "model.layers.63.mlp.experts.54.up_proj", "model.layers.63.mlp.experts.55.up_proj", "model.layers.63.mlp.experts.56.up_proj", "model.layers.63.mlp.experts.57.up_proj", "model.layers.63.mlp.experts.58.up_proj", "model.layers.63.mlp.experts.59.up_proj", "model.layers.63.mlp.experts.60.up_proj", "model.layers.63.mlp.experts.61.up_proj", "model.layers.63.mlp.experts.62.up_proj", "model.layers.63.mlp.experts.63.up_proj", "model.layers.63.mlp.experts.64.up_proj", "model.layers.63.mlp.experts.65.up_proj", "model.layers.63.mlp.experts.66.up_proj", "model.layers.63.mlp.experts.67.up_proj", "model.layers.63.mlp.experts.68.up_proj", "model.layers.63.mlp.experts.69.up_proj", "model.layers.63.mlp.experts.70.up_proj", "model.layers.63.mlp.experts.71.up_proj", "model.layers.63.mlp.experts.72.up_proj", "model.layers.63.mlp.experts.73.up_proj", "model.layers.63.mlp.experts.74.up_proj", "model.layers.63.mlp.experts.75.up_proj", "model.layers.63.mlp.experts.76.up_proj", "model.layers.63.mlp.experts.77.up_proj", "model.layers.63.mlp.experts.78.up_proj", "model.layers.63.mlp.experts.79.up_proj", "model.layers.63.mlp.experts.80.up_proj", "model.layers.63.mlp.experts.81.up_proj", "model.layers.63.mlp.experts.82.up_proj", "model.layers.63.mlp.experts.83.up_proj", "model.layers.63.mlp.experts.84.up_proj", "model.layers.63.mlp.experts.85.up_proj", "model.layers.63.mlp.experts.86.up_proj", "model.layers.63.mlp.experts.87.up_proj", "model.layers.63.mlp.experts.88.up_proj", "model.layers.63.mlp.experts.89.up_proj", "model.layers.63.mlp.experts.90.up_proj", "model.layers.63.mlp.experts.91.up_proj", "model.layers.63.mlp.experts.92.up_proj", "model.layers.63.mlp.experts.93.up_proj", "model.layers.63.mlp.experts.94.up_proj", "model.layers.63.mlp.experts.95.up_proj", "model.layers.63.mlp.experts.96.up_proj", "model.layers.63.mlp.experts.97.up_proj", "model.layers.63.mlp.experts.98.up_proj", "model.layers.63.mlp.experts.99.up_proj", "model.layers.63.mlp.experts.100.up_proj", "model.layers.63.mlp.experts.101.up_proj", "model.layers.63.mlp.experts.102.up_proj", "model.layers.63.mlp.experts.103.up_proj", "model.layers.63.mlp.experts.104.up_proj", "model.layers.63.mlp.experts.105.up_proj", "model.layers.63.mlp.experts.106.up_proj", "model.layers.63.mlp.experts.107.up_proj", "model.layers.63.mlp.experts.108.up_proj", "model.layers.63.mlp.experts.109.up_proj", "model.layers.63.mlp.experts.110.up_proj", "model.layers.63.mlp.experts.111.up_proj", "model.layers.63.mlp.experts.112.up_proj", "model.layers.63.mlp.experts.113.up_proj", "model.layers.63.mlp.experts.114.up_proj", "model.layers.63.mlp.experts.115.up_proj", "model.layers.63.mlp.experts.116.up_proj", "model.layers.63.mlp.experts.117.up_proj", "model.layers.63.mlp.experts.118.up_proj", "model.layers.63.mlp.experts.119.up_proj", "model.layers.63.mlp.experts.120.up_proj", "model.layers.63.mlp.experts.121.up_proj", "model.layers.63.mlp.experts.122.up_proj", "model.layers.63.mlp.experts.123.up_proj", "model.layers.63.mlp.experts.124.up_proj", "model.layers.63.mlp.experts.125.up_proj", "model.layers.63.mlp.experts.126.up_proj", "model.layers.63.mlp.experts.127.up_proj", "model.layers.63.mlp.experts.128.up_proj", "model.layers.63.mlp.experts.129.up_proj", "model.layers.63.mlp.experts.130.up_proj", "model.layers.63.mlp.experts.131.up_proj", "model.layers.63.mlp.experts.132.up_proj", "model.layers.63.mlp.experts.133.up_proj", "model.layers.63.mlp.experts.134.up_proj", "model.layers.63.mlp.experts.135.up_proj", "model.layers.63.mlp.experts.136.up_proj", "model.layers.63.mlp.experts.137.up_proj", "model.layers.63.mlp.experts.138.up_proj", "model.layers.63.mlp.experts.139.up_proj", "model.layers.63.mlp.experts.140.up_proj", "model.layers.63.mlp.experts.141.up_proj", "model.layers.63.mlp.experts.142.up_proj", "model.layers.63.mlp.experts.143.up_proj", "model.layers.63.mlp.experts.144.up_proj", "model.layers.63.mlp.experts.145.up_proj", "model.layers.63.mlp.experts.146.up_proj", "model.layers.63.mlp.experts.147.up_proj", "model.layers.63.mlp.experts.148.up_proj", "model.layers.63.mlp.experts.149.up_proj", "model.layers.63.mlp.experts.150.up_proj", "model.layers.63.mlp.experts.151.up_proj", "model.layers.63.mlp.experts.152.up_proj", "model.layers.63.mlp.experts.153.up_proj", "model.layers.63.mlp.experts.154.up_proj", "model.layers.63.mlp.experts.155.up_proj", "model.layers.63.mlp.experts.156.up_proj", "model.layers.63.mlp.experts.157.up_proj", "model.layers.63.mlp.experts.158.up_proj", "model.layers.63.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -2.835091436282017e-07, "dbits": 5033164800 } ] }, { "idx": 380, "layers": [ "model.layers.63.mlp.experts.0.down_proj", "model.layers.63.mlp.experts.1.down_proj", "model.layers.63.mlp.experts.2.down_proj", "model.layers.63.mlp.experts.3.down_proj", "model.layers.63.mlp.experts.4.down_proj", "model.layers.63.mlp.experts.5.down_proj", "model.layers.63.mlp.experts.6.down_proj", "model.layers.63.mlp.experts.7.down_proj", "model.layers.63.mlp.experts.8.down_proj", "model.layers.63.mlp.experts.9.down_proj", "model.layers.63.mlp.experts.10.down_proj", "model.layers.63.mlp.experts.11.down_proj", "model.layers.63.mlp.experts.12.down_proj", "model.layers.63.mlp.experts.13.down_proj", "model.layers.63.mlp.experts.14.down_proj", "model.layers.63.mlp.experts.15.down_proj", "model.layers.63.mlp.experts.16.down_proj", "model.layers.63.mlp.experts.17.down_proj", "model.layers.63.mlp.experts.18.down_proj", "model.layers.63.mlp.experts.19.down_proj", "model.layers.63.mlp.experts.20.down_proj", "model.layers.63.mlp.experts.21.down_proj", "model.layers.63.mlp.experts.22.down_proj", "model.layers.63.mlp.experts.23.down_proj", "model.layers.63.mlp.experts.24.down_proj", "model.layers.63.mlp.experts.25.down_proj", "model.layers.63.mlp.experts.26.down_proj", "model.layers.63.mlp.experts.27.down_proj", "model.layers.63.mlp.experts.28.down_proj", "model.layers.63.mlp.experts.29.down_proj", "model.layers.63.mlp.experts.30.down_proj", "model.layers.63.mlp.experts.31.down_proj", "model.layers.63.mlp.experts.32.down_proj", "model.layers.63.mlp.experts.33.down_proj", "model.layers.63.mlp.experts.34.down_proj", "model.layers.63.mlp.experts.35.down_proj", "model.layers.63.mlp.experts.36.down_proj", "model.layers.63.mlp.experts.37.down_proj", "model.layers.63.mlp.experts.38.down_proj", "model.layers.63.mlp.experts.39.down_proj", "model.layers.63.mlp.experts.40.down_proj", "model.layers.63.mlp.experts.41.down_proj", "model.layers.63.mlp.experts.42.down_proj", "model.layers.63.mlp.experts.43.down_proj", "model.layers.63.mlp.experts.44.down_proj", "model.layers.63.mlp.experts.45.down_proj", "model.layers.63.mlp.experts.46.down_proj", "model.layers.63.mlp.experts.47.down_proj", "model.layers.63.mlp.experts.48.down_proj", "model.layers.63.mlp.experts.49.down_proj", "model.layers.63.mlp.experts.50.down_proj", "model.layers.63.mlp.experts.51.down_proj", "model.layers.63.mlp.experts.52.down_proj", "model.layers.63.mlp.experts.53.down_proj", "model.layers.63.mlp.experts.54.down_proj", "model.layers.63.mlp.experts.55.down_proj", "model.layers.63.mlp.experts.56.down_proj", "model.layers.63.mlp.experts.57.down_proj", "model.layers.63.mlp.experts.58.down_proj", "model.layers.63.mlp.experts.59.down_proj", "model.layers.63.mlp.experts.60.down_proj", "model.layers.63.mlp.experts.61.down_proj", "model.layers.63.mlp.experts.62.down_proj", "model.layers.63.mlp.experts.63.down_proj", "model.layers.63.mlp.experts.64.down_proj", "model.layers.63.mlp.experts.65.down_proj", "model.layers.63.mlp.experts.66.down_proj", "model.layers.63.mlp.experts.67.down_proj", "model.layers.63.mlp.experts.68.down_proj", "model.layers.63.mlp.experts.69.down_proj", "model.layers.63.mlp.experts.70.down_proj", "model.layers.63.mlp.experts.71.down_proj", "model.layers.63.mlp.experts.72.down_proj", "model.layers.63.mlp.experts.73.down_proj", "model.layers.63.mlp.experts.74.down_proj", "model.layers.63.mlp.experts.75.down_proj", "model.layers.63.mlp.experts.76.down_proj", "model.layers.63.mlp.experts.77.down_proj", "model.layers.63.mlp.experts.78.down_proj", "model.layers.63.mlp.experts.79.down_proj", "model.layers.63.mlp.experts.80.down_proj", "model.layers.63.mlp.experts.81.down_proj", "model.layers.63.mlp.experts.82.down_proj", "model.layers.63.mlp.experts.83.down_proj", "model.layers.63.mlp.experts.84.down_proj", "model.layers.63.mlp.experts.85.down_proj", "model.layers.63.mlp.experts.86.down_proj", "model.layers.63.mlp.experts.87.down_proj", "model.layers.63.mlp.experts.88.down_proj", "model.layers.63.mlp.experts.89.down_proj", "model.layers.63.mlp.experts.90.down_proj", "model.layers.63.mlp.experts.91.down_proj", "model.layers.63.mlp.experts.92.down_proj", "model.layers.63.mlp.experts.93.down_proj", "model.layers.63.mlp.experts.94.down_proj", "model.layers.63.mlp.experts.95.down_proj", "model.layers.63.mlp.experts.96.down_proj", "model.layers.63.mlp.experts.97.down_proj", "model.layers.63.mlp.experts.98.down_proj", "model.layers.63.mlp.experts.99.down_proj", "model.layers.63.mlp.experts.100.down_proj", "model.layers.63.mlp.experts.101.down_proj", "model.layers.63.mlp.experts.102.down_proj", "model.layers.63.mlp.experts.103.down_proj", "model.layers.63.mlp.experts.104.down_proj", "model.layers.63.mlp.experts.105.down_proj", "model.layers.63.mlp.experts.106.down_proj", "model.layers.63.mlp.experts.107.down_proj", "model.layers.63.mlp.experts.108.down_proj", "model.layers.63.mlp.experts.109.down_proj", "model.layers.63.mlp.experts.110.down_proj", "model.layers.63.mlp.experts.111.down_proj", "model.layers.63.mlp.experts.112.down_proj", "model.layers.63.mlp.experts.113.down_proj", "model.layers.63.mlp.experts.114.down_proj", "model.layers.63.mlp.experts.115.down_proj", "model.layers.63.mlp.experts.116.down_proj", "model.layers.63.mlp.experts.117.down_proj", "model.layers.63.mlp.experts.118.down_proj", "model.layers.63.mlp.experts.119.down_proj", "model.layers.63.mlp.experts.120.down_proj", "model.layers.63.mlp.experts.121.down_proj", "model.layers.63.mlp.experts.122.down_proj", "model.layers.63.mlp.experts.123.down_proj", "model.layers.63.mlp.experts.124.down_proj", "model.layers.63.mlp.experts.125.down_proj", "model.layers.63.mlp.experts.126.down_proj", "model.layers.63.mlp.experts.127.down_proj", "model.layers.63.mlp.experts.128.down_proj", "model.layers.63.mlp.experts.129.down_proj", "model.layers.63.mlp.experts.130.down_proj", "model.layers.63.mlp.experts.131.down_proj", "model.layers.63.mlp.experts.132.down_proj", "model.layers.63.mlp.experts.133.down_proj", "model.layers.63.mlp.experts.134.down_proj", "model.layers.63.mlp.experts.135.down_proj", "model.layers.63.mlp.experts.136.down_proj", "model.layers.63.mlp.experts.137.down_proj", "model.layers.63.mlp.experts.138.down_proj", "model.layers.63.mlp.experts.139.down_proj", "model.layers.63.mlp.experts.140.down_proj", "model.layers.63.mlp.experts.141.down_proj", "model.layers.63.mlp.experts.142.down_proj", "model.layers.63.mlp.experts.143.down_proj", "model.layers.63.mlp.experts.144.down_proj", "model.layers.63.mlp.experts.145.down_proj", "model.layers.63.mlp.experts.146.down_proj", "model.layers.63.mlp.experts.147.down_proj", "model.layers.63.mlp.experts.148.down_proj", "model.layers.63.mlp.experts.149.down_proj", "model.layers.63.mlp.experts.150.down_proj", "model.layers.63.mlp.experts.151.down_proj", "model.layers.63.mlp.experts.152.down_proj", "model.layers.63.mlp.experts.153.down_proj", "model.layers.63.mlp.experts.154.down_proj", "model.layers.63.mlp.experts.155.down_proj", "model.layers.63.mlp.experts.156.down_proj", "model.layers.63.mlp.experts.157.down_proj", "model.layers.63.mlp.experts.158.down_proj", "model.layers.63.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 5.382782546803355e-06, "dbits": 2516582400 } ] }, { "idx": 381, "layers": [ "model.layers.64.self_attn.q_proj" ], "candidates": [ { "dkld": 1.9225757569074414e-06, "dbits": 125829120 } ] }, { "idx": 382, "layers": [ "model.layers.64.self_attn.k_proj", "model.layers.64.self_attn.v_proj" ], "candidates": [ { "dkld": 2.9458198696375327e-06, "dbits": 20971520 } ] }, { "idx": 383, "layers": [ "model.layers.64.self_attn.o_proj" ], "candidates": [ { "dkld": 4.039885243400888e-06, "dbits": 125829120 } ] }, { "idx": 384, "layers": [ "model.layers.64.mlp.shared_experts.gate_proj", "model.layers.64.mlp.shared_experts.up_proj", "model.layers.64.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.497612386942343e-07, "dbits": 47185920 } ] }, { "idx": 385, "layers": [ "model.layers.64.mlp.experts.0.gate_proj", "model.layers.64.mlp.experts.1.gate_proj", "model.layers.64.mlp.experts.2.gate_proj", "model.layers.64.mlp.experts.3.gate_proj", "model.layers.64.mlp.experts.4.gate_proj", "model.layers.64.mlp.experts.5.gate_proj", "model.layers.64.mlp.experts.6.gate_proj", "model.layers.64.mlp.experts.7.gate_proj", "model.layers.64.mlp.experts.8.gate_proj", "model.layers.64.mlp.experts.9.gate_proj", "model.layers.64.mlp.experts.10.gate_proj", "model.layers.64.mlp.experts.11.gate_proj", "model.layers.64.mlp.experts.12.gate_proj", "model.layers.64.mlp.experts.13.gate_proj", "model.layers.64.mlp.experts.14.gate_proj", "model.layers.64.mlp.experts.15.gate_proj", "model.layers.64.mlp.experts.16.gate_proj", "model.layers.64.mlp.experts.17.gate_proj", "model.layers.64.mlp.experts.18.gate_proj", "model.layers.64.mlp.experts.19.gate_proj", "model.layers.64.mlp.experts.20.gate_proj", "model.layers.64.mlp.experts.21.gate_proj", "model.layers.64.mlp.experts.22.gate_proj", "model.layers.64.mlp.experts.23.gate_proj", "model.layers.64.mlp.experts.24.gate_proj", "model.layers.64.mlp.experts.25.gate_proj", "model.layers.64.mlp.experts.26.gate_proj", "model.layers.64.mlp.experts.27.gate_proj", "model.layers.64.mlp.experts.28.gate_proj", "model.layers.64.mlp.experts.29.gate_proj", "model.layers.64.mlp.experts.30.gate_proj", "model.layers.64.mlp.experts.31.gate_proj", "model.layers.64.mlp.experts.32.gate_proj", "model.layers.64.mlp.experts.33.gate_proj", "model.layers.64.mlp.experts.34.gate_proj", "model.layers.64.mlp.experts.35.gate_proj", "model.layers.64.mlp.experts.36.gate_proj", "model.layers.64.mlp.experts.37.gate_proj", "model.layers.64.mlp.experts.38.gate_proj", "model.layers.64.mlp.experts.39.gate_proj", "model.layers.64.mlp.experts.40.gate_proj", "model.layers.64.mlp.experts.41.gate_proj", "model.layers.64.mlp.experts.42.gate_proj", "model.layers.64.mlp.experts.43.gate_proj", "model.layers.64.mlp.experts.44.gate_proj", "model.layers.64.mlp.experts.45.gate_proj", "model.layers.64.mlp.experts.46.gate_proj", "model.layers.64.mlp.experts.47.gate_proj", "model.layers.64.mlp.experts.48.gate_proj", "model.layers.64.mlp.experts.49.gate_proj", "model.layers.64.mlp.experts.50.gate_proj", "model.layers.64.mlp.experts.51.gate_proj", "model.layers.64.mlp.experts.52.gate_proj", "model.layers.64.mlp.experts.53.gate_proj", "model.layers.64.mlp.experts.54.gate_proj", "model.layers.64.mlp.experts.55.gate_proj", "model.layers.64.mlp.experts.56.gate_proj", "model.layers.64.mlp.experts.57.gate_proj", "model.layers.64.mlp.experts.58.gate_proj", "model.layers.64.mlp.experts.59.gate_proj", "model.layers.64.mlp.experts.60.gate_proj", "model.layers.64.mlp.experts.61.gate_proj", "model.layers.64.mlp.experts.62.gate_proj", "model.layers.64.mlp.experts.63.gate_proj", "model.layers.64.mlp.experts.64.gate_proj", "model.layers.64.mlp.experts.65.gate_proj", "model.layers.64.mlp.experts.66.gate_proj", "model.layers.64.mlp.experts.67.gate_proj", "model.layers.64.mlp.experts.68.gate_proj", "model.layers.64.mlp.experts.69.gate_proj", "model.layers.64.mlp.experts.70.gate_proj", "model.layers.64.mlp.experts.71.gate_proj", "model.layers.64.mlp.experts.72.gate_proj", "model.layers.64.mlp.experts.73.gate_proj", "model.layers.64.mlp.experts.74.gate_proj", "model.layers.64.mlp.experts.75.gate_proj", "model.layers.64.mlp.experts.76.gate_proj", "model.layers.64.mlp.experts.77.gate_proj", "model.layers.64.mlp.experts.78.gate_proj", "model.layers.64.mlp.experts.79.gate_proj", "model.layers.64.mlp.experts.80.gate_proj", "model.layers.64.mlp.experts.81.gate_proj", "model.layers.64.mlp.experts.82.gate_proj", "model.layers.64.mlp.experts.83.gate_proj", "model.layers.64.mlp.experts.84.gate_proj", "model.layers.64.mlp.experts.85.gate_proj", "model.layers.64.mlp.experts.86.gate_proj", "model.layers.64.mlp.experts.87.gate_proj", "model.layers.64.mlp.experts.88.gate_proj", "model.layers.64.mlp.experts.89.gate_proj", "model.layers.64.mlp.experts.90.gate_proj", "model.layers.64.mlp.experts.91.gate_proj", "model.layers.64.mlp.experts.92.gate_proj", "model.layers.64.mlp.experts.93.gate_proj", "model.layers.64.mlp.experts.94.gate_proj", "model.layers.64.mlp.experts.95.gate_proj", "model.layers.64.mlp.experts.96.gate_proj", "model.layers.64.mlp.experts.97.gate_proj", "model.layers.64.mlp.experts.98.gate_proj", "model.layers.64.mlp.experts.99.gate_proj", "model.layers.64.mlp.experts.100.gate_proj", "model.layers.64.mlp.experts.101.gate_proj", "model.layers.64.mlp.experts.102.gate_proj", "model.layers.64.mlp.experts.103.gate_proj", "model.layers.64.mlp.experts.104.gate_proj", "model.layers.64.mlp.experts.105.gate_proj", "model.layers.64.mlp.experts.106.gate_proj", "model.layers.64.mlp.experts.107.gate_proj", "model.layers.64.mlp.experts.108.gate_proj", "model.layers.64.mlp.experts.109.gate_proj", "model.layers.64.mlp.experts.110.gate_proj", "model.layers.64.mlp.experts.111.gate_proj", "model.layers.64.mlp.experts.112.gate_proj", "model.layers.64.mlp.experts.113.gate_proj", "model.layers.64.mlp.experts.114.gate_proj", "model.layers.64.mlp.experts.115.gate_proj", "model.layers.64.mlp.experts.116.gate_proj", "model.layers.64.mlp.experts.117.gate_proj", "model.layers.64.mlp.experts.118.gate_proj", "model.layers.64.mlp.experts.119.gate_proj", "model.layers.64.mlp.experts.120.gate_proj", "model.layers.64.mlp.experts.121.gate_proj", "model.layers.64.mlp.experts.122.gate_proj", "model.layers.64.mlp.experts.123.gate_proj", "model.layers.64.mlp.experts.124.gate_proj", "model.layers.64.mlp.experts.125.gate_proj", "model.layers.64.mlp.experts.126.gate_proj", "model.layers.64.mlp.experts.127.gate_proj", "model.layers.64.mlp.experts.128.gate_proj", "model.layers.64.mlp.experts.129.gate_proj", "model.layers.64.mlp.experts.130.gate_proj", "model.layers.64.mlp.experts.131.gate_proj", "model.layers.64.mlp.experts.132.gate_proj", "model.layers.64.mlp.experts.133.gate_proj", "model.layers.64.mlp.experts.134.gate_proj", "model.layers.64.mlp.experts.135.gate_proj", "model.layers.64.mlp.experts.136.gate_proj", "model.layers.64.mlp.experts.137.gate_proj", "model.layers.64.mlp.experts.138.gate_proj", "model.layers.64.mlp.experts.139.gate_proj", "model.layers.64.mlp.experts.140.gate_proj", "model.layers.64.mlp.experts.141.gate_proj", "model.layers.64.mlp.experts.142.gate_proj", "model.layers.64.mlp.experts.143.gate_proj", "model.layers.64.mlp.experts.144.gate_proj", "model.layers.64.mlp.experts.145.gate_proj", "model.layers.64.mlp.experts.146.gate_proj", "model.layers.64.mlp.experts.147.gate_proj", "model.layers.64.mlp.experts.148.gate_proj", "model.layers.64.mlp.experts.149.gate_proj", "model.layers.64.mlp.experts.150.gate_proj", "model.layers.64.mlp.experts.151.gate_proj", "model.layers.64.mlp.experts.152.gate_proj", "model.layers.64.mlp.experts.153.gate_proj", "model.layers.64.mlp.experts.154.gate_proj", "model.layers.64.mlp.experts.155.gate_proj", "model.layers.64.mlp.experts.156.gate_proj", "model.layers.64.mlp.experts.157.gate_proj", "model.layers.64.mlp.experts.158.gate_proj", "model.layers.64.mlp.experts.159.gate_proj", "model.layers.64.mlp.experts.0.up_proj", "model.layers.64.mlp.experts.1.up_proj", "model.layers.64.mlp.experts.2.up_proj", "model.layers.64.mlp.experts.3.up_proj", "model.layers.64.mlp.experts.4.up_proj", "model.layers.64.mlp.experts.5.up_proj", "model.layers.64.mlp.experts.6.up_proj", "model.layers.64.mlp.experts.7.up_proj", "model.layers.64.mlp.experts.8.up_proj", "model.layers.64.mlp.experts.9.up_proj", "model.layers.64.mlp.experts.10.up_proj", "model.layers.64.mlp.experts.11.up_proj", "model.layers.64.mlp.experts.12.up_proj", "model.layers.64.mlp.experts.13.up_proj", "model.layers.64.mlp.experts.14.up_proj", "model.layers.64.mlp.experts.15.up_proj", "model.layers.64.mlp.experts.16.up_proj", "model.layers.64.mlp.experts.17.up_proj", "model.layers.64.mlp.experts.18.up_proj", "model.layers.64.mlp.experts.19.up_proj", "model.layers.64.mlp.experts.20.up_proj", "model.layers.64.mlp.experts.21.up_proj", "model.layers.64.mlp.experts.22.up_proj", "model.layers.64.mlp.experts.23.up_proj", "model.layers.64.mlp.experts.24.up_proj", "model.layers.64.mlp.experts.25.up_proj", "model.layers.64.mlp.experts.26.up_proj", "model.layers.64.mlp.experts.27.up_proj", "model.layers.64.mlp.experts.28.up_proj", "model.layers.64.mlp.experts.29.up_proj", "model.layers.64.mlp.experts.30.up_proj", "model.layers.64.mlp.experts.31.up_proj", "model.layers.64.mlp.experts.32.up_proj", "model.layers.64.mlp.experts.33.up_proj", "model.layers.64.mlp.experts.34.up_proj", "model.layers.64.mlp.experts.35.up_proj", "model.layers.64.mlp.experts.36.up_proj", "model.layers.64.mlp.experts.37.up_proj", "model.layers.64.mlp.experts.38.up_proj", "model.layers.64.mlp.experts.39.up_proj", "model.layers.64.mlp.experts.40.up_proj", "model.layers.64.mlp.experts.41.up_proj", "model.layers.64.mlp.experts.42.up_proj", "model.layers.64.mlp.experts.43.up_proj", "model.layers.64.mlp.experts.44.up_proj", "model.layers.64.mlp.experts.45.up_proj", "model.layers.64.mlp.experts.46.up_proj", "model.layers.64.mlp.experts.47.up_proj", "model.layers.64.mlp.experts.48.up_proj", "model.layers.64.mlp.experts.49.up_proj", "model.layers.64.mlp.experts.50.up_proj", "model.layers.64.mlp.experts.51.up_proj", "model.layers.64.mlp.experts.52.up_proj", "model.layers.64.mlp.experts.53.up_proj", "model.layers.64.mlp.experts.54.up_proj", "model.layers.64.mlp.experts.55.up_proj", "model.layers.64.mlp.experts.56.up_proj", "model.layers.64.mlp.experts.57.up_proj", "model.layers.64.mlp.experts.58.up_proj", "model.layers.64.mlp.experts.59.up_proj", "model.layers.64.mlp.experts.60.up_proj", "model.layers.64.mlp.experts.61.up_proj", "model.layers.64.mlp.experts.62.up_proj", "model.layers.64.mlp.experts.63.up_proj", "model.layers.64.mlp.experts.64.up_proj", "model.layers.64.mlp.experts.65.up_proj", "model.layers.64.mlp.experts.66.up_proj", "model.layers.64.mlp.experts.67.up_proj", "model.layers.64.mlp.experts.68.up_proj", "model.layers.64.mlp.experts.69.up_proj", "model.layers.64.mlp.experts.70.up_proj", "model.layers.64.mlp.experts.71.up_proj", "model.layers.64.mlp.experts.72.up_proj", "model.layers.64.mlp.experts.73.up_proj", "model.layers.64.mlp.experts.74.up_proj", "model.layers.64.mlp.experts.75.up_proj", "model.layers.64.mlp.experts.76.up_proj", "model.layers.64.mlp.experts.77.up_proj", "model.layers.64.mlp.experts.78.up_proj", "model.layers.64.mlp.experts.79.up_proj", "model.layers.64.mlp.experts.80.up_proj", "model.layers.64.mlp.experts.81.up_proj", "model.layers.64.mlp.experts.82.up_proj", "model.layers.64.mlp.experts.83.up_proj", "model.layers.64.mlp.experts.84.up_proj", "model.layers.64.mlp.experts.85.up_proj", "model.layers.64.mlp.experts.86.up_proj", "model.layers.64.mlp.experts.87.up_proj", "model.layers.64.mlp.experts.88.up_proj", "model.layers.64.mlp.experts.89.up_proj", "model.layers.64.mlp.experts.90.up_proj", "model.layers.64.mlp.experts.91.up_proj", "model.layers.64.mlp.experts.92.up_proj", "model.layers.64.mlp.experts.93.up_proj", "model.layers.64.mlp.experts.94.up_proj", "model.layers.64.mlp.experts.95.up_proj", "model.layers.64.mlp.experts.96.up_proj", "model.layers.64.mlp.experts.97.up_proj", "model.layers.64.mlp.experts.98.up_proj", "model.layers.64.mlp.experts.99.up_proj", "model.layers.64.mlp.experts.100.up_proj", "model.layers.64.mlp.experts.101.up_proj", "model.layers.64.mlp.experts.102.up_proj", "model.layers.64.mlp.experts.103.up_proj", "model.layers.64.mlp.experts.104.up_proj", "model.layers.64.mlp.experts.105.up_proj", "model.layers.64.mlp.experts.106.up_proj", "model.layers.64.mlp.experts.107.up_proj", "model.layers.64.mlp.experts.108.up_proj", "model.layers.64.mlp.experts.109.up_proj", "model.layers.64.mlp.experts.110.up_proj", "model.layers.64.mlp.experts.111.up_proj", "model.layers.64.mlp.experts.112.up_proj", "model.layers.64.mlp.experts.113.up_proj", "model.layers.64.mlp.experts.114.up_proj", "model.layers.64.mlp.experts.115.up_proj", "model.layers.64.mlp.experts.116.up_proj", "model.layers.64.mlp.experts.117.up_proj", "model.layers.64.mlp.experts.118.up_proj", "model.layers.64.mlp.experts.119.up_proj", "model.layers.64.mlp.experts.120.up_proj", "model.layers.64.mlp.experts.121.up_proj", "model.layers.64.mlp.experts.122.up_proj", "model.layers.64.mlp.experts.123.up_proj", "model.layers.64.mlp.experts.124.up_proj", "model.layers.64.mlp.experts.125.up_proj", "model.layers.64.mlp.experts.126.up_proj", "model.layers.64.mlp.experts.127.up_proj", "model.layers.64.mlp.experts.128.up_proj", "model.layers.64.mlp.experts.129.up_proj", "model.layers.64.mlp.experts.130.up_proj", "model.layers.64.mlp.experts.131.up_proj", "model.layers.64.mlp.experts.132.up_proj", "model.layers.64.mlp.experts.133.up_proj", "model.layers.64.mlp.experts.134.up_proj", "model.layers.64.mlp.experts.135.up_proj", "model.layers.64.mlp.experts.136.up_proj", "model.layers.64.mlp.experts.137.up_proj", "model.layers.64.mlp.experts.138.up_proj", "model.layers.64.mlp.experts.139.up_proj", "model.layers.64.mlp.experts.140.up_proj", "model.layers.64.mlp.experts.141.up_proj", "model.layers.64.mlp.experts.142.up_proj", "model.layers.64.mlp.experts.143.up_proj", "model.layers.64.mlp.experts.144.up_proj", "model.layers.64.mlp.experts.145.up_proj", "model.layers.64.mlp.experts.146.up_proj", "model.layers.64.mlp.experts.147.up_proj", "model.layers.64.mlp.experts.148.up_proj", "model.layers.64.mlp.experts.149.up_proj", "model.layers.64.mlp.experts.150.up_proj", "model.layers.64.mlp.experts.151.up_proj", "model.layers.64.mlp.experts.152.up_proj", "model.layers.64.mlp.experts.153.up_proj", "model.layers.64.mlp.experts.154.up_proj", "model.layers.64.mlp.experts.155.up_proj", "model.layers.64.mlp.experts.156.up_proj", "model.layers.64.mlp.experts.157.up_proj", "model.layers.64.mlp.experts.158.up_proj", "model.layers.64.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 9.361392585560679e-07, "dbits": 5033164800 } ] }, { "idx": 386, "layers": [ "model.layers.64.mlp.experts.0.down_proj", "model.layers.64.mlp.experts.1.down_proj", "model.layers.64.mlp.experts.2.down_proj", "model.layers.64.mlp.experts.3.down_proj", "model.layers.64.mlp.experts.4.down_proj", "model.layers.64.mlp.experts.5.down_proj", "model.layers.64.mlp.experts.6.down_proj", "model.layers.64.mlp.experts.7.down_proj", "model.layers.64.mlp.experts.8.down_proj", "model.layers.64.mlp.experts.9.down_proj", "model.layers.64.mlp.experts.10.down_proj", "model.layers.64.mlp.experts.11.down_proj", "model.layers.64.mlp.experts.12.down_proj", "model.layers.64.mlp.experts.13.down_proj", "model.layers.64.mlp.experts.14.down_proj", "model.layers.64.mlp.experts.15.down_proj", "model.layers.64.mlp.experts.16.down_proj", "model.layers.64.mlp.experts.17.down_proj", "model.layers.64.mlp.experts.18.down_proj", "model.layers.64.mlp.experts.19.down_proj", "model.layers.64.mlp.experts.20.down_proj", "model.layers.64.mlp.experts.21.down_proj", "model.layers.64.mlp.experts.22.down_proj", "model.layers.64.mlp.experts.23.down_proj", "model.layers.64.mlp.experts.24.down_proj", "model.layers.64.mlp.experts.25.down_proj", "model.layers.64.mlp.experts.26.down_proj", "model.layers.64.mlp.experts.27.down_proj", "model.layers.64.mlp.experts.28.down_proj", "model.layers.64.mlp.experts.29.down_proj", "model.layers.64.mlp.experts.30.down_proj", "model.layers.64.mlp.experts.31.down_proj", "model.layers.64.mlp.experts.32.down_proj", "model.layers.64.mlp.experts.33.down_proj", "model.layers.64.mlp.experts.34.down_proj", "model.layers.64.mlp.experts.35.down_proj", "model.layers.64.mlp.experts.36.down_proj", "model.layers.64.mlp.experts.37.down_proj", "model.layers.64.mlp.experts.38.down_proj", "model.layers.64.mlp.experts.39.down_proj", "model.layers.64.mlp.experts.40.down_proj", "model.layers.64.mlp.experts.41.down_proj", "model.layers.64.mlp.experts.42.down_proj", "model.layers.64.mlp.experts.43.down_proj", "model.layers.64.mlp.experts.44.down_proj", "model.layers.64.mlp.experts.45.down_proj", "model.layers.64.mlp.experts.46.down_proj", "model.layers.64.mlp.experts.47.down_proj", "model.layers.64.mlp.experts.48.down_proj", "model.layers.64.mlp.experts.49.down_proj", "model.layers.64.mlp.experts.50.down_proj", "model.layers.64.mlp.experts.51.down_proj", "model.layers.64.mlp.experts.52.down_proj", "model.layers.64.mlp.experts.53.down_proj", "model.layers.64.mlp.experts.54.down_proj", "model.layers.64.mlp.experts.55.down_proj", "model.layers.64.mlp.experts.56.down_proj", "model.layers.64.mlp.experts.57.down_proj", "model.layers.64.mlp.experts.58.down_proj", "model.layers.64.mlp.experts.59.down_proj", "model.layers.64.mlp.experts.60.down_proj", "model.layers.64.mlp.experts.61.down_proj", "model.layers.64.mlp.experts.62.down_proj", "model.layers.64.mlp.experts.63.down_proj", "model.layers.64.mlp.experts.64.down_proj", "model.layers.64.mlp.experts.65.down_proj", "model.layers.64.mlp.experts.66.down_proj", "model.layers.64.mlp.experts.67.down_proj", "model.layers.64.mlp.experts.68.down_proj", "model.layers.64.mlp.experts.69.down_proj", "model.layers.64.mlp.experts.70.down_proj", "model.layers.64.mlp.experts.71.down_proj", "model.layers.64.mlp.experts.72.down_proj", "model.layers.64.mlp.experts.73.down_proj", "model.layers.64.mlp.experts.74.down_proj", "model.layers.64.mlp.experts.75.down_proj", "model.layers.64.mlp.experts.76.down_proj", "model.layers.64.mlp.experts.77.down_proj", "model.layers.64.mlp.experts.78.down_proj", "model.layers.64.mlp.experts.79.down_proj", "model.layers.64.mlp.experts.80.down_proj", "model.layers.64.mlp.experts.81.down_proj", "model.layers.64.mlp.experts.82.down_proj", "model.layers.64.mlp.experts.83.down_proj", "model.layers.64.mlp.experts.84.down_proj", "model.layers.64.mlp.experts.85.down_proj", "model.layers.64.mlp.experts.86.down_proj", "model.layers.64.mlp.experts.87.down_proj", "model.layers.64.mlp.experts.88.down_proj", "model.layers.64.mlp.experts.89.down_proj", "model.layers.64.mlp.experts.90.down_proj", "model.layers.64.mlp.experts.91.down_proj", "model.layers.64.mlp.experts.92.down_proj", "model.layers.64.mlp.experts.93.down_proj", "model.layers.64.mlp.experts.94.down_proj", "model.layers.64.mlp.experts.95.down_proj", "model.layers.64.mlp.experts.96.down_proj", "model.layers.64.mlp.experts.97.down_proj", "model.layers.64.mlp.experts.98.down_proj", "model.layers.64.mlp.experts.99.down_proj", "model.layers.64.mlp.experts.100.down_proj", "model.layers.64.mlp.experts.101.down_proj", "model.layers.64.mlp.experts.102.down_proj", "model.layers.64.mlp.experts.103.down_proj", "model.layers.64.mlp.experts.104.down_proj", "model.layers.64.mlp.experts.105.down_proj", "model.layers.64.mlp.experts.106.down_proj", "model.layers.64.mlp.experts.107.down_proj", "model.layers.64.mlp.experts.108.down_proj", "model.layers.64.mlp.experts.109.down_proj", "model.layers.64.mlp.experts.110.down_proj", "model.layers.64.mlp.experts.111.down_proj", "model.layers.64.mlp.experts.112.down_proj", "model.layers.64.mlp.experts.113.down_proj", "model.layers.64.mlp.experts.114.down_proj", "model.layers.64.mlp.experts.115.down_proj", "model.layers.64.mlp.experts.116.down_proj", "model.layers.64.mlp.experts.117.down_proj", "model.layers.64.mlp.experts.118.down_proj", "model.layers.64.mlp.experts.119.down_proj", "model.layers.64.mlp.experts.120.down_proj", "model.layers.64.mlp.experts.121.down_proj", "model.layers.64.mlp.experts.122.down_proj", "model.layers.64.mlp.experts.123.down_proj", "model.layers.64.mlp.experts.124.down_proj", "model.layers.64.mlp.experts.125.down_proj", "model.layers.64.mlp.experts.126.down_proj", "model.layers.64.mlp.experts.127.down_proj", "model.layers.64.mlp.experts.128.down_proj", "model.layers.64.mlp.experts.129.down_proj", "model.layers.64.mlp.experts.130.down_proj", "model.layers.64.mlp.experts.131.down_proj", "model.layers.64.mlp.experts.132.down_proj", "model.layers.64.mlp.experts.133.down_proj", "model.layers.64.mlp.experts.134.down_proj", "model.layers.64.mlp.experts.135.down_proj", "model.layers.64.mlp.experts.136.down_proj", "model.layers.64.mlp.experts.137.down_proj", "model.layers.64.mlp.experts.138.down_proj", "model.layers.64.mlp.experts.139.down_proj", "model.layers.64.mlp.experts.140.down_proj", "model.layers.64.mlp.experts.141.down_proj", "model.layers.64.mlp.experts.142.down_proj", "model.layers.64.mlp.experts.143.down_proj", "model.layers.64.mlp.experts.144.down_proj", "model.layers.64.mlp.experts.145.down_proj", "model.layers.64.mlp.experts.146.down_proj", "model.layers.64.mlp.experts.147.down_proj", "model.layers.64.mlp.experts.148.down_proj", "model.layers.64.mlp.experts.149.down_proj", "model.layers.64.mlp.experts.150.down_proj", "model.layers.64.mlp.experts.151.down_proj", "model.layers.64.mlp.experts.152.down_proj", "model.layers.64.mlp.experts.153.down_proj", "model.layers.64.mlp.experts.154.down_proj", "model.layers.64.mlp.experts.155.down_proj", "model.layers.64.mlp.experts.156.down_proj", "model.layers.64.mlp.experts.157.down_proj", "model.layers.64.mlp.experts.158.down_proj", "model.layers.64.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.8785056201740883e-06, "dbits": 2516582400 } ] }, { "idx": 387, "layers": [ "model.layers.65.self_attn.q_proj" ], "candidates": [ { "dkld": 2.4471461074426987e-06, "dbits": 125829120 } ] }, { "idx": 388, "layers": [ "model.layers.65.self_attn.k_proj", "model.layers.65.self_attn.v_proj" ], "candidates": [ { "dkld": -6.434973329305215e-07, "dbits": 20971520 } ] }, { "idx": 389, "layers": [ "model.layers.65.self_attn.o_proj" ], "candidates": [ { "dkld": 2.2192049073055603e-06, "dbits": 125829120 } ] }, { "idx": 390, "layers": [ "model.layers.65.mlp.shared_experts.gate_proj", "model.layers.65.mlp.shared_experts.up_proj", "model.layers.65.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.9704981241375425e-06, "dbits": 47185920 } ] }, { "idx": 391, "layers": [ "model.layers.65.mlp.experts.0.gate_proj", "model.layers.65.mlp.experts.1.gate_proj", "model.layers.65.mlp.experts.2.gate_proj", "model.layers.65.mlp.experts.3.gate_proj", "model.layers.65.mlp.experts.4.gate_proj", "model.layers.65.mlp.experts.5.gate_proj", "model.layers.65.mlp.experts.6.gate_proj", "model.layers.65.mlp.experts.7.gate_proj", "model.layers.65.mlp.experts.8.gate_proj", "model.layers.65.mlp.experts.9.gate_proj", "model.layers.65.mlp.experts.10.gate_proj", "model.layers.65.mlp.experts.11.gate_proj", "model.layers.65.mlp.experts.12.gate_proj", "model.layers.65.mlp.experts.13.gate_proj", "model.layers.65.mlp.experts.14.gate_proj", "model.layers.65.mlp.experts.15.gate_proj", "model.layers.65.mlp.experts.16.gate_proj", "model.layers.65.mlp.experts.17.gate_proj", "model.layers.65.mlp.experts.18.gate_proj", "model.layers.65.mlp.experts.19.gate_proj", "model.layers.65.mlp.experts.20.gate_proj", "model.layers.65.mlp.experts.21.gate_proj", "model.layers.65.mlp.experts.22.gate_proj", "model.layers.65.mlp.experts.23.gate_proj", "model.layers.65.mlp.experts.24.gate_proj", "model.layers.65.mlp.experts.25.gate_proj", "model.layers.65.mlp.experts.26.gate_proj", "model.layers.65.mlp.experts.27.gate_proj", "model.layers.65.mlp.experts.28.gate_proj", "model.layers.65.mlp.experts.29.gate_proj", "model.layers.65.mlp.experts.30.gate_proj", "model.layers.65.mlp.experts.31.gate_proj", "model.layers.65.mlp.experts.32.gate_proj", "model.layers.65.mlp.experts.33.gate_proj", "model.layers.65.mlp.experts.34.gate_proj", "model.layers.65.mlp.experts.35.gate_proj", "model.layers.65.mlp.experts.36.gate_proj", "model.layers.65.mlp.experts.37.gate_proj", "model.layers.65.mlp.experts.38.gate_proj", "model.layers.65.mlp.experts.39.gate_proj", "model.layers.65.mlp.experts.40.gate_proj", "model.layers.65.mlp.experts.41.gate_proj", "model.layers.65.mlp.experts.42.gate_proj", "model.layers.65.mlp.experts.43.gate_proj", "model.layers.65.mlp.experts.44.gate_proj", "model.layers.65.mlp.experts.45.gate_proj", "model.layers.65.mlp.experts.46.gate_proj", "model.layers.65.mlp.experts.47.gate_proj", "model.layers.65.mlp.experts.48.gate_proj", "model.layers.65.mlp.experts.49.gate_proj", "model.layers.65.mlp.experts.50.gate_proj", "model.layers.65.mlp.experts.51.gate_proj", "model.layers.65.mlp.experts.52.gate_proj", "model.layers.65.mlp.experts.53.gate_proj", "model.layers.65.mlp.experts.54.gate_proj", "model.layers.65.mlp.experts.55.gate_proj", "model.layers.65.mlp.experts.56.gate_proj", "model.layers.65.mlp.experts.57.gate_proj", "model.layers.65.mlp.experts.58.gate_proj", "model.layers.65.mlp.experts.59.gate_proj", "model.layers.65.mlp.experts.60.gate_proj", "model.layers.65.mlp.experts.61.gate_proj", "model.layers.65.mlp.experts.62.gate_proj", "model.layers.65.mlp.experts.63.gate_proj", "model.layers.65.mlp.experts.64.gate_proj", "model.layers.65.mlp.experts.65.gate_proj", "model.layers.65.mlp.experts.66.gate_proj", "model.layers.65.mlp.experts.67.gate_proj", "model.layers.65.mlp.experts.68.gate_proj", "model.layers.65.mlp.experts.69.gate_proj", "model.layers.65.mlp.experts.70.gate_proj", "model.layers.65.mlp.experts.71.gate_proj", "model.layers.65.mlp.experts.72.gate_proj", "model.layers.65.mlp.experts.73.gate_proj", "model.layers.65.mlp.experts.74.gate_proj", "model.layers.65.mlp.experts.75.gate_proj", "model.layers.65.mlp.experts.76.gate_proj", "model.layers.65.mlp.experts.77.gate_proj", "model.layers.65.mlp.experts.78.gate_proj", "model.layers.65.mlp.experts.79.gate_proj", "model.layers.65.mlp.experts.80.gate_proj", "model.layers.65.mlp.experts.81.gate_proj", "model.layers.65.mlp.experts.82.gate_proj", "model.layers.65.mlp.experts.83.gate_proj", "model.layers.65.mlp.experts.84.gate_proj", "model.layers.65.mlp.experts.85.gate_proj", "model.layers.65.mlp.experts.86.gate_proj", "model.layers.65.mlp.experts.87.gate_proj", "model.layers.65.mlp.experts.88.gate_proj", "model.layers.65.mlp.experts.89.gate_proj", "model.layers.65.mlp.experts.90.gate_proj", "model.layers.65.mlp.experts.91.gate_proj", "model.layers.65.mlp.experts.92.gate_proj", "model.layers.65.mlp.experts.93.gate_proj", "model.layers.65.mlp.experts.94.gate_proj", "model.layers.65.mlp.experts.95.gate_proj", "model.layers.65.mlp.experts.96.gate_proj", "model.layers.65.mlp.experts.97.gate_proj", "model.layers.65.mlp.experts.98.gate_proj", "model.layers.65.mlp.experts.99.gate_proj", "model.layers.65.mlp.experts.100.gate_proj", "model.layers.65.mlp.experts.101.gate_proj", "model.layers.65.mlp.experts.102.gate_proj", "model.layers.65.mlp.experts.103.gate_proj", "model.layers.65.mlp.experts.104.gate_proj", "model.layers.65.mlp.experts.105.gate_proj", "model.layers.65.mlp.experts.106.gate_proj", "model.layers.65.mlp.experts.107.gate_proj", "model.layers.65.mlp.experts.108.gate_proj", "model.layers.65.mlp.experts.109.gate_proj", "model.layers.65.mlp.experts.110.gate_proj", "model.layers.65.mlp.experts.111.gate_proj", "model.layers.65.mlp.experts.112.gate_proj", "model.layers.65.mlp.experts.113.gate_proj", "model.layers.65.mlp.experts.114.gate_proj", "model.layers.65.mlp.experts.115.gate_proj", "model.layers.65.mlp.experts.116.gate_proj", "model.layers.65.mlp.experts.117.gate_proj", "model.layers.65.mlp.experts.118.gate_proj", "model.layers.65.mlp.experts.119.gate_proj", "model.layers.65.mlp.experts.120.gate_proj", "model.layers.65.mlp.experts.121.gate_proj", "model.layers.65.mlp.experts.122.gate_proj", "model.layers.65.mlp.experts.123.gate_proj", "model.layers.65.mlp.experts.124.gate_proj", "model.layers.65.mlp.experts.125.gate_proj", "model.layers.65.mlp.experts.126.gate_proj", "model.layers.65.mlp.experts.127.gate_proj", "model.layers.65.mlp.experts.128.gate_proj", "model.layers.65.mlp.experts.129.gate_proj", "model.layers.65.mlp.experts.130.gate_proj", "model.layers.65.mlp.experts.131.gate_proj", "model.layers.65.mlp.experts.132.gate_proj", "model.layers.65.mlp.experts.133.gate_proj", "model.layers.65.mlp.experts.134.gate_proj", "model.layers.65.mlp.experts.135.gate_proj", "model.layers.65.mlp.experts.136.gate_proj", "model.layers.65.mlp.experts.137.gate_proj", "model.layers.65.mlp.experts.138.gate_proj", "model.layers.65.mlp.experts.139.gate_proj", "model.layers.65.mlp.experts.140.gate_proj", "model.layers.65.mlp.experts.141.gate_proj", "model.layers.65.mlp.experts.142.gate_proj", "model.layers.65.mlp.experts.143.gate_proj", "model.layers.65.mlp.experts.144.gate_proj", "model.layers.65.mlp.experts.145.gate_proj", "model.layers.65.mlp.experts.146.gate_proj", "model.layers.65.mlp.experts.147.gate_proj", "model.layers.65.mlp.experts.148.gate_proj", "model.layers.65.mlp.experts.149.gate_proj", "model.layers.65.mlp.experts.150.gate_proj", "model.layers.65.mlp.experts.151.gate_proj", "model.layers.65.mlp.experts.152.gate_proj", "model.layers.65.mlp.experts.153.gate_proj", "model.layers.65.mlp.experts.154.gate_proj", "model.layers.65.mlp.experts.155.gate_proj", "model.layers.65.mlp.experts.156.gate_proj", "model.layers.65.mlp.experts.157.gate_proj", "model.layers.65.mlp.experts.158.gate_proj", "model.layers.65.mlp.experts.159.gate_proj", "model.layers.65.mlp.experts.0.up_proj", "model.layers.65.mlp.experts.1.up_proj", "model.layers.65.mlp.experts.2.up_proj", "model.layers.65.mlp.experts.3.up_proj", "model.layers.65.mlp.experts.4.up_proj", "model.layers.65.mlp.experts.5.up_proj", "model.layers.65.mlp.experts.6.up_proj", "model.layers.65.mlp.experts.7.up_proj", "model.layers.65.mlp.experts.8.up_proj", "model.layers.65.mlp.experts.9.up_proj", "model.layers.65.mlp.experts.10.up_proj", "model.layers.65.mlp.experts.11.up_proj", "model.layers.65.mlp.experts.12.up_proj", "model.layers.65.mlp.experts.13.up_proj", "model.layers.65.mlp.experts.14.up_proj", "model.layers.65.mlp.experts.15.up_proj", "model.layers.65.mlp.experts.16.up_proj", "model.layers.65.mlp.experts.17.up_proj", "model.layers.65.mlp.experts.18.up_proj", "model.layers.65.mlp.experts.19.up_proj", "model.layers.65.mlp.experts.20.up_proj", "model.layers.65.mlp.experts.21.up_proj", "model.layers.65.mlp.experts.22.up_proj", "model.layers.65.mlp.experts.23.up_proj", "model.layers.65.mlp.experts.24.up_proj", "model.layers.65.mlp.experts.25.up_proj", "model.layers.65.mlp.experts.26.up_proj", "model.layers.65.mlp.experts.27.up_proj", "model.layers.65.mlp.experts.28.up_proj", "model.layers.65.mlp.experts.29.up_proj", "model.layers.65.mlp.experts.30.up_proj", "model.layers.65.mlp.experts.31.up_proj", "model.layers.65.mlp.experts.32.up_proj", "model.layers.65.mlp.experts.33.up_proj", "model.layers.65.mlp.experts.34.up_proj", "model.layers.65.mlp.experts.35.up_proj", "model.layers.65.mlp.experts.36.up_proj", "model.layers.65.mlp.experts.37.up_proj", "model.layers.65.mlp.experts.38.up_proj", "model.layers.65.mlp.experts.39.up_proj", "model.layers.65.mlp.experts.40.up_proj", "model.layers.65.mlp.experts.41.up_proj", "model.layers.65.mlp.experts.42.up_proj", "model.layers.65.mlp.experts.43.up_proj", "model.layers.65.mlp.experts.44.up_proj", "model.layers.65.mlp.experts.45.up_proj", "model.layers.65.mlp.experts.46.up_proj", "model.layers.65.mlp.experts.47.up_proj", "model.layers.65.mlp.experts.48.up_proj", "model.layers.65.mlp.experts.49.up_proj", "model.layers.65.mlp.experts.50.up_proj", "model.layers.65.mlp.experts.51.up_proj", "model.layers.65.mlp.experts.52.up_proj", "model.layers.65.mlp.experts.53.up_proj", "model.layers.65.mlp.experts.54.up_proj", "model.layers.65.mlp.experts.55.up_proj", "model.layers.65.mlp.experts.56.up_proj", "model.layers.65.mlp.experts.57.up_proj", "model.layers.65.mlp.experts.58.up_proj", "model.layers.65.mlp.experts.59.up_proj", "model.layers.65.mlp.experts.60.up_proj", "model.layers.65.mlp.experts.61.up_proj", "model.layers.65.mlp.experts.62.up_proj", "model.layers.65.mlp.experts.63.up_proj", "model.layers.65.mlp.experts.64.up_proj", "model.layers.65.mlp.experts.65.up_proj", "model.layers.65.mlp.experts.66.up_proj", "model.layers.65.mlp.experts.67.up_proj", "model.layers.65.mlp.experts.68.up_proj", "model.layers.65.mlp.experts.69.up_proj", "model.layers.65.mlp.experts.70.up_proj", "model.layers.65.mlp.experts.71.up_proj", "model.layers.65.mlp.experts.72.up_proj", "model.layers.65.mlp.experts.73.up_proj", "model.layers.65.mlp.experts.74.up_proj", "model.layers.65.mlp.experts.75.up_proj", "model.layers.65.mlp.experts.76.up_proj", "model.layers.65.mlp.experts.77.up_proj", "model.layers.65.mlp.experts.78.up_proj", "model.layers.65.mlp.experts.79.up_proj", "model.layers.65.mlp.experts.80.up_proj", "model.layers.65.mlp.experts.81.up_proj", "model.layers.65.mlp.experts.82.up_proj", "model.layers.65.mlp.experts.83.up_proj", "model.layers.65.mlp.experts.84.up_proj", "model.layers.65.mlp.experts.85.up_proj", "model.layers.65.mlp.experts.86.up_proj", "model.layers.65.mlp.experts.87.up_proj", "model.layers.65.mlp.experts.88.up_proj", "model.layers.65.mlp.experts.89.up_proj", "model.layers.65.mlp.experts.90.up_proj", "model.layers.65.mlp.experts.91.up_proj", "model.layers.65.mlp.experts.92.up_proj", "model.layers.65.mlp.experts.93.up_proj", "model.layers.65.mlp.experts.94.up_proj", "model.layers.65.mlp.experts.95.up_proj", "model.layers.65.mlp.experts.96.up_proj", "model.layers.65.mlp.experts.97.up_proj", "model.layers.65.mlp.experts.98.up_proj", "model.layers.65.mlp.experts.99.up_proj", "model.layers.65.mlp.experts.100.up_proj", "model.layers.65.mlp.experts.101.up_proj", "model.layers.65.mlp.experts.102.up_proj", "model.layers.65.mlp.experts.103.up_proj", "model.layers.65.mlp.experts.104.up_proj", "model.layers.65.mlp.experts.105.up_proj", "model.layers.65.mlp.experts.106.up_proj", "model.layers.65.mlp.experts.107.up_proj", "model.layers.65.mlp.experts.108.up_proj", "model.layers.65.mlp.experts.109.up_proj", "model.layers.65.mlp.experts.110.up_proj", "model.layers.65.mlp.experts.111.up_proj", "model.layers.65.mlp.experts.112.up_proj", "model.layers.65.mlp.experts.113.up_proj", "model.layers.65.mlp.experts.114.up_proj", "model.layers.65.mlp.experts.115.up_proj", "model.layers.65.mlp.experts.116.up_proj", "model.layers.65.mlp.experts.117.up_proj", "model.layers.65.mlp.experts.118.up_proj", "model.layers.65.mlp.experts.119.up_proj", "model.layers.65.mlp.experts.120.up_proj", "model.layers.65.mlp.experts.121.up_proj", "model.layers.65.mlp.experts.122.up_proj", "model.layers.65.mlp.experts.123.up_proj", "model.layers.65.mlp.experts.124.up_proj", "model.layers.65.mlp.experts.125.up_proj", "model.layers.65.mlp.experts.126.up_proj", "model.layers.65.mlp.experts.127.up_proj", "model.layers.65.mlp.experts.128.up_proj", "model.layers.65.mlp.experts.129.up_proj", "model.layers.65.mlp.experts.130.up_proj", "model.layers.65.mlp.experts.131.up_proj", "model.layers.65.mlp.experts.132.up_proj", "model.layers.65.mlp.experts.133.up_proj", "model.layers.65.mlp.experts.134.up_proj", "model.layers.65.mlp.experts.135.up_proj", "model.layers.65.mlp.experts.136.up_proj", "model.layers.65.mlp.experts.137.up_proj", "model.layers.65.mlp.experts.138.up_proj", "model.layers.65.mlp.experts.139.up_proj", "model.layers.65.mlp.experts.140.up_proj", "model.layers.65.mlp.experts.141.up_proj", "model.layers.65.mlp.experts.142.up_proj", "model.layers.65.mlp.experts.143.up_proj", "model.layers.65.mlp.experts.144.up_proj", "model.layers.65.mlp.experts.145.up_proj", "model.layers.65.mlp.experts.146.up_proj", "model.layers.65.mlp.experts.147.up_proj", "model.layers.65.mlp.experts.148.up_proj", "model.layers.65.mlp.experts.149.up_proj", "model.layers.65.mlp.experts.150.up_proj", "model.layers.65.mlp.experts.151.up_proj", "model.layers.65.mlp.experts.152.up_proj", "model.layers.65.mlp.experts.153.up_proj", "model.layers.65.mlp.experts.154.up_proj", "model.layers.65.mlp.experts.155.up_proj", "model.layers.65.mlp.experts.156.up_proj", "model.layers.65.mlp.experts.157.up_proj", "model.layers.65.mlp.experts.158.up_proj", "model.layers.65.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.621951565146663e-07, "dbits": 5033164800 } ] }, { "idx": 392, "layers": [ "model.layers.65.mlp.experts.0.down_proj", "model.layers.65.mlp.experts.1.down_proj", "model.layers.65.mlp.experts.2.down_proj", "model.layers.65.mlp.experts.3.down_proj", "model.layers.65.mlp.experts.4.down_proj", "model.layers.65.mlp.experts.5.down_proj", "model.layers.65.mlp.experts.6.down_proj", "model.layers.65.mlp.experts.7.down_proj", "model.layers.65.mlp.experts.8.down_proj", "model.layers.65.mlp.experts.9.down_proj", "model.layers.65.mlp.experts.10.down_proj", "model.layers.65.mlp.experts.11.down_proj", "model.layers.65.mlp.experts.12.down_proj", "model.layers.65.mlp.experts.13.down_proj", "model.layers.65.mlp.experts.14.down_proj", "model.layers.65.mlp.experts.15.down_proj", "model.layers.65.mlp.experts.16.down_proj", "model.layers.65.mlp.experts.17.down_proj", "model.layers.65.mlp.experts.18.down_proj", "model.layers.65.mlp.experts.19.down_proj", "model.layers.65.mlp.experts.20.down_proj", "model.layers.65.mlp.experts.21.down_proj", "model.layers.65.mlp.experts.22.down_proj", "model.layers.65.mlp.experts.23.down_proj", "model.layers.65.mlp.experts.24.down_proj", "model.layers.65.mlp.experts.25.down_proj", "model.layers.65.mlp.experts.26.down_proj", "model.layers.65.mlp.experts.27.down_proj", "model.layers.65.mlp.experts.28.down_proj", "model.layers.65.mlp.experts.29.down_proj", "model.layers.65.mlp.experts.30.down_proj", "model.layers.65.mlp.experts.31.down_proj", "model.layers.65.mlp.experts.32.down_proj", "model.layers.65.mlp.experts.33.down_proj", "model.layers.65.mlp.experts.34.down_proj", "model.layers.65.mlp.experts.35.down_proj", "model.layers.65.mlp.experts.36.down_proj", "model.layers.65.mlp.experts.37.down_proj", "model.layers.65.mlp.experts.38.down_proj", "model.layers.65.mlp.experts.39.down_proj", "model.layers.65.mlp.experts.40.down_proj", "model.layers.65.mlp.experts.41.down_proj", "model.layers.65.mlp.experts.42.down_proj", "model.layers.65.mlp.experts.43.down_proj", "model.layers.65.mlp.experts.44.down_proj", "model.layers.65.mlp.experts.45.down_proj", "model.layers.65.mlp.experts.46.down_proj", "model.layers.65.mlp.experts.47.down_proj", "model.layers.65.mlp.experts.48.down_proj", "model.layers.65.mlp.experts.49.down_proj", "model.layers.65.mlp.experts.50.down_proj", "model.layers.65.mlp.experts.51.down_proj", "model.layers.65.mlp.experts.52.down_proj", "model.layers.65.mlp.experts.53.down_proj", "model.layers.65.mlp.experts.54.down_proj", "model.layers.65.mlp.experts.55.down_proj", "model.layers.65.mlp.experts.56.down_proj", "model.layers.65.mlp.experts.57.down_proj", "model.layers.65.mlp.experts.58.down_proj", "model.layers.65.mlp.experts.59.down_proj", "model.layers.65.mlp.experts.60.down_proj", "model.layers.65.mlp.experts.61.down_proj", "model.layers.65.mlp.experts.62.down_proj", "model.layers.65.mlp.experts.63.down_proj", "model.layers.65.mlp.experts.64.down_proj", "model.layers.65.mlp.experts.65.down_proj", "model.layers.65.mlp.experts.66.down_proj", "model.layers.65.mlp.experts.67.down_proj", "model.layers.65.mlp.experts.68.down_proj", "model.layers.65.mlp.experts.69.down_proj", "model.layers.65.mlp.experts.70.down_proj", "model.layers.65.mlp.experts.71.down_proj", "model.layers.65.mlp.experts.72.down_proj", "model.layers.65.mlp.experts.73.down_proj", "model.layers.65.mlp.experts.74.down_proj", "model.layers.65.mlp.experts.75.down_proj", "model.layers.65.mlp.experts.76.down_proj", "model.layers.65.mlp.experts.77.down_proj", "model.layers.65.mlp.experts.78.down_proj", "model.layers.65.mlp.experts.79.down_proj", "model.layers.65.mlp.experts.80.down_proj", "model.layers.65.mlp.experts.81.down_proj", "model.layers.65.mlp.experts.82.down_proj", "model.layers.65.mlp.experts.83.down_proj", "model.layers.65.mlp.experts.84.down_proj", "model.layers.65.mlp.experts.85.down_proj", "model.layers.65.mlp.experts.86.down_proj", "model.layers.65.mlp.experts.87.down_proj", "model.layers.65.mlp.experts.88.down_proj", "model.layers.65.mlp.experts.89.down_proj", "model.layers.65.mlp.experts.90.down_proj", "model.layers.65.mlp.experts.91.down_proj", "model.layers.65.mlp.experts.92.down_proj", "model.layers.65.mlp.experts.93.down_proj", "model.layers.65.mlp.experts.94.down_proj", "model.layers.65.mlp.experts.95.down_proj", "model.layers.65.mlp.experts.96.down_proj", "model.layers.65.mlp.experts.97.down_proj", "model.layers.65.mlp.experts.98.down_proj", "model.layers.65.mlp.experts.99.down_proj", "model.layers.65.mlp.experts.100.down_proj", "model.layers.65.mlp.experts.101.down_proj", "model.layers.65.mlp.experts.102.down_proj", "model.layers.65.mlp.experts.103.down_proj", "model.layers.65.mlp.experts.104.down_proj", "model.layers.65.mlp.experts.105.down_proj", "model.layers.65.mlp.experts.106.down_proj", "model.layers.65.mlp.experts.107.down_proj", "model.layers.65.mlp.experts.108.down_proj", "model.layers.65.mlp.experts.109.down_proj", "model.layers.65.mlp.experts.110.down_proj", "model.layers.65.mlp.experts.111.down_proj", "model.layers.65.mlp.experts.112.down_proj", "model.layers.65.mlp.experts.113.down_proj", "model.layers.65.mlp.experts.114.down_proj", "model.layers.65.mlp.experts.115.down_proj", "model.layers.65.mlp.experts.116.down_proj", "model.layers.65.mlp.experts.117.down_proj", "model.layers.65.mlp.experts.118.down_proj", "model.layers.65.mlp.experts.119.down_proj", "model.layers.65.mlp.experts.120.down_proj", "model.layers.65.mlp.experts.121.down_proj", "model.layers.65.mlp.experts.122.down_proj", "model.layers.65.mlp.experts.123.down_proj", "model.layers.65.mlp.experts.124.down_proj", "model.layers.65.mlp.experts.125.down_proj", "model.layers.65.mlp.experts.126.down_proj", "model.layers.65.mlp.experts.127.down_proj", "model.layers.65.mlp.experts.128.down_proj", "model.layers.65.mlp.experts.129.down_proj", "model.layers.65.mlp.experts.130.down_proj", "model.layers.65.mlp.experts.131.down_proj", "model.layers.65.mlp.experts.132.down_proj", "model.layers.65.mlp.experts.133.down_proj", "model.layers.65.mlp.experts.134.down_proj", "model.layers.65.mlp.experts.135.down_proj", "model.layers.65.mlp.experts.136.down_proj", "model.layers.65.mlp.experts.137.down_proj", "model.layers.65.mlp.experts.138.down_proj", "model.layers.65.mlp.experts.139.down_proj", "model.layers.65.mlp.experts.140.down_proj", "model.layers.65.mlp.experts.141.down_proj", "model.layers.65.mlp.experts.142.down_proj", "model.layers.65.mlp.experts.143.down_proj", "model.layers.65.mlp.experts.144.down_proj", "model.layers.65.mlp.experts.145.down_proj", "model.layers.65.mlp.experts.146.down_proj", "model.layers.65.mlp.experts.147.down_proj", "model.layers.65.mlp.experts.148.down_proj", "model.layers.65.mlp.experts.149.down_proj", "model.layers.65.mlp.experts.150.down_proj", "model.layers.65.mlp.experts.151.down_proj", "model.layers.65.mlp.experts.152.down_proj", "model.layers.65.mlp.experts.153.down_proj", "model.layers.65.mlp.experts.154.down_proj", "model.layers.65.mlp.experts.155.down_proj", "model.layers.65.mlp.experts.156.down_proj", "model.layers.65.mlp.experts.157.down_proj", "model.layers.65.mlp.experts.158.down_proj", "model.layers.65.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.137988435104468e-06, "dbits": 2516582400 } ] }, { "idx": 393, "layers": [ "model.layers.66.self_attn.q_proj" ], "candidates": [ { "dkld": -7.573107723146677e-07, "dbits": 125829120 } ] }, { "idx": 394, "layers": [ "model.layers.66.self_attn.k_proj", "model.layers.66.self_attn.v_proj" ], "candidates": [ { "dkld": 5.482218693941875e-06, "dbits": 20971520 } ] }, { "idx": 395, "layers": [ "model.layers.66.self_attn.o_proj" ], "candidates": [ { "dkld": 3.1665142159909444e-06, "dbits": 125829120 } ] }, { "idx": 396, "layers": [ "model.layers.66.mlp.shared_experts.gate_proj", "model.layers.66.mlp.shared_experts.up_proj", "model.layers.66.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.0952022168785767e-06, "dbits": 47185920 } ] }, { "idx": 397, "layers": [ "model.layers.66.mlp.experts.0.gate_proj", "model.layers.66.mlp.experts.1.gate_proj", "model.layers.66.mlp.experts.2.gate_proj", "model.layers.66.mlp.experts.3.gate_proj", "model.layers.66.mlp.experts.4.gate_proj", "model.layers.66.mlp.experts.5.gate_proj", "model.layers.66.mlp.experts.6.gate_proj", "model.layers.66.mlp.experts.7.gate_proj", "model.layers.66.mlp.experts.8.gate_proj", "model.layers.66.mlp.experts.9.gate_proj", "model.layers.66.mlp.experts.10.gate_proj", "model.layers.66.mlp.experts.11.gate_proj", "model.layers.66.mlp.experts.12.gate_proj", "model.layers.66.mlp.experts.13.gate_proj", "model.layers.66.mlp.experts.14.gate_proj", "model.layers.66.mlp.experts.15.gate_proj", "model.layers.66.mlp.experts.16.gate_proj", "model.layers.66.mlp.experts.17.gate_proj", "model.layers.66.mlp.experts.18.gate_proj", "model.layers.66.mlp.experts.19.gate_proj", "model.layers.66.mlp.experts.20.gate_proj", "model.layers.66.mlp.experts.21.gate_proj", "model.layers.66.mlp.experts.22.gate_proj", "model.layers.66.mlp.experts.23.gate_proj", "model.layers.66.mlp.experts.24.gate_proj", "model.layers.66.mlp.experts.25.gate_proj", "model.layers.66.mlp.experts.26.gate_proj", "model.layers.66.mlp.experts.27.gate_proj", "model.layers.66.mlp.experts.28.gate_proj", "model.layers.66.mlp.experts.29.gate_proj", "model.layers.66.mlp.experts.30.gate_proj", "model.layers.66.mlp.experts.31.gate_proj", "model.layers.66.mlp.experts.32.gate_proj", "model.layers.66.mlp.experts.33.gate_proj", "model.layers.66.mlp.experts.34.gate_proj", "model.layers.66.mlp.experts.35.gate_proj", "model.layers.66.mlp.experts.36.gate_proj", "model.layers.66.mlp.experts.37.gate_proj", "model.layers.66.mlp.experts.38.gate_proj", "model.layers.66.mlp.experts.39.gate_proj", "model.layers.66.mlp.experts.40.gate_proj", "model.layers.66.mlp.experts.41.gate_proj", "model.layers.66.mlp.experts.42.gate_proj", "model.layers.66.mlp.experts.43.gate_proj", "model.layers.66.mlp.experts.44.gate_proj", "model.layers.66.mlp.experts.45.gate_proj", "model.layers.66.mlp.experts.46.gate_proj", "model.layers.66.mlp.experts.47.gate_proj", "model.layers.66.mlp.experts.48.gate_proj", "model.layers.66.mlp.experts.49.gate_proj", "model.layers.66.mlp.experts.50.gate_proj", "model.layers.66.mlp.experts.51.gate_proj", "model.layers.66.mlp.experts.52.gate_proj", "model.layers.66.mlp.experts.53.gate_proj", "model.layers.66.mlp.experts.54.gate_proj", "model.layers.66.mlp.experts.55.gate_proj", "model.layers.66.mlp.experts.56.gate_proj", "model.layers.66.mlp.experts.57.gate_proj", "model.layers.66.mlp.experts.58.gate_proj", "model.layers.66.mlp.experts.59.gate_proj", "model.layers.66.mlp.experts.60.gate_proj", "model.layers.66.mlp.experts.61.gate_proj", "model.layers.66.mlp.experts.62.gate_proj", "model.layers.66.mlp.experts.63.gate_proj", "model.layers.66.mlp.experts.64.gate_proj", "model.layers.66.mlp.experts.65.gate_proj", "model.layers.66.mlp.experts.66.gate_proj", "model.layers.66.mlp.experts.67.gate_proj", "model.layers.66.mlp.experts.68.gate_proj", "model.layers.66.mlp.experts.69.gate_proj", "model.layers.66.mlp.experts.70.gate_proj", "model.layers.66.mlp.experts.71.gate_proj", "model.layers.66.mlp.experts.72.gate_proj", "model.layers.66.mlp.experts.73.gate_proj", "model.layers.66.mlp.experts.74.gate_proj", "model.layers.66.mlp.experts.75.gate_proj", "model.layers.66.mlp.experts.76.gate_proj", "model.layers.66.mlp.experts.77.gate_proj", "model.layers.66.mlp.experts.78.gate_proj", "model.layers.66.mlp.experts.79.gate_proj", "model.layers.66.mlp.experts.80.gate_proj", "model.layers.66.mlp.experts.81.gate_proj", "model.layers.66.mlp.experts.82.gate_proj", "model.layers.66.mlp.experts.83.gate_proj", "model.layers.66.mlp.experts.84.gate_proj", "model.layers.66.mlp.experts.85.gate_proj", "model.layers.66.mlp.experts.86.gate_proj", "model.layers.66.mlp.experts.87.gate_proj", "model.layers.66.mlp.experts.88.gate_proj", "model.layers.66.mlp.experts.89.gate_proj", "model.layers.66.mlp.experts.90.gate_proj", "model.layers.66.mlp.experts.91.gate_proj", "model.layers.66.mlp.experts.92.gate_proj", "model.layers.66.mlp.experts.93.gate_proj", "model.layers.66.mlp.experts.94.gate_proj", "model.layers.66.mlp.experts.95.gate_proj", "model.layers.66.mlp.experts.96.gate_proj", "model.layers.66.mlp.experts.97.gate_proj", "model.layers.66.mlp.experts.98.gate_proj", "model.layers.66.mlp.experts.99.gate_proj", "model.layers.66.mlp.experts.100.gate_proj", "model.layers.66.mlp.experts.101.gate_proj", "model.layers.66.mlp.experts.102.gate_proj", "model.layers.66.mlp.experts.103.gate_proj", "model.layers.66.mlp.experts.104.gate_proj", "model.layers.66.mlp.experts.105.gate_proj", "model.layers.66.mlp.experts.106.gate_proj", "model.layers.66.mlp.experts.107.gate_proj", "model.layers.66.mlp.experts.108.gate_proj", "model.layers.66.mlp.experts.109.gate_proj", "model.layers.66.mlp.experts.110.gate_proj", "model.layers.66.mlp.experts.111.gate_proj", "model.layers.66.mlp.experts.112.gate_proj", "model.layers.66.mlp.experts.113.gate_proj", "model.layers.66.mlp.experts.114.gate_proj", "model.layers.66.mlp.experts.115.gate_proj", "model.layers.66.mlp.experts.116.gate_proj", "model.layers.66.mlp.experts.117.gate_proj", "model.layers.66.mlp.experts.118.gate_proj", "model.layers.66.mlp.experts.119.gate_proj", "model.layers.66.mlp.experts.120.gate_proj", "model.layers.66.mlp.experts.121.gate_proj", "model.layers.66.mlp.experts.122.gate_proj", "model.layers.66.mlp.experts.123.gate_proj", "model.layers.66.mlp.experts.124.gate_proj", "model.layers.66.mlp.experts.125.gate_proj", "model.layers.66.mlp.experts.126.gate_proj", "model.layers.66.mlp.experts.127.gate_proj", "model.layers.66.mlp.experts.128.gate_proj", "model.layers.66.mlp.experts.129.gate_proj", "model.layers.66.mlp.experts.130.gate_proj", "model.layers.66.mlp.experts.131.gate_proj", "model.layers.66.mlp.experts.132.gate_proj", "model.layers.66.mlp.experts.133.gate_proj", "model.layers.66.mlp.experts.134.gate_proj", "model.layers.66.mlp.experts.135.gate_proj", "model.layers.66.mlp.experts.136.gate_proj", "model.layers.66.mlp.experts.137.gate_proj", "model.layers.66.mlp.experts.138.gate_proj", "model.layers.66.mlp.experts.139.gate_proj", "model.layers.66.mlp.experts.140.gate_proj", "model.layers.66.mlp.experts.141.gate_proj", "model.layers.66.mlp.experts.142.gate_proj", "model.layers.66.mlp.experts.143.gate_proj", "model.layers.66.mlp.experts.144.gate_proj", "model.layers.66.mlp.experts.145.gate_proj", "model.layers.66.mlp.experts.146.gate_proj", "model.layers.66.mlp.experts.147.gate_proj", "model.layers.66.mlp.experts.148.gate_proj", "model.layers.66.mlp.experts.149.gate_proj", "model.layers.66.mlp.experts.150.gate_proj", "model.layers.66.mlp.experts.151.gate_proj", "model.layers.66.mlp.experts.152.gate_proj", "model.layers.66.mlp.experts.153.gate_proj", "model.layers.66.mlp.experts.154.gate_proj", "model.layers.66.mlp.experts.155.gate_proj", "model.layers.66.mlp.experts.156.gate_proj", "model.layers.66.mlp.experts.157.gate_proj", "model.layers.66.mlp.experts.158.gate_proj", "model.layers.66.mlp.experts.159.gate_proj", "model.layers.66.mlp.experts.0.up_proj", "model.layers.66.mlp.experts.1.up_proj", "model.layers.66.mlp.experts.2.up_proj", "model.layers.66.mlp.experts.3.up_proj", "model.layers.66.mlp.experts.4.up_proj", "model.layers.66.mlp.experts.5.up_proj", "model.layers.66.mlp.experts.6.up_proj", "model.layers.66.mlp.experts.7.up_proj", "model.layers.66.mlp.experts.8.up_proj", "model.layers.66.mlp.experts.9.up_proj", "model.layers.66.mlp.experts.10.up_proj", "model.layers.66.mlp.experts.11.up_proj", "model.layers.66.mlp.experts.12.up_proj", "model.layers.66.mlp.experts.13.up_proj", "model.layers.66.mlp.experts.14.up_proj", "model.layers.66.mlp.experts.15.up_proj", "model.layers.66.mlp.experts.16.up_proj", "model.layers.66.mlp.experts.17.up_proj", "model.layers.66.mlp.experts.18.up_proj", "model.layers.66.mlp.experts.19.up_proj", "model.layers.66.mlp.experts.20.up_proj", "model.layers.66.mlp.experts.21.up_proj", "model.layers.66.mlp.experts.22.up_proj", "model.layers.66.mlp.experts.23.up_proj", "model.layers.66.mlp.experts.24.up_proj", "model.layers.66.mlp.experts.25.up_proj", "model.layers.66.mlp.experts.26.up_proj", "model.layers.66.mlp.experts.27.up_proj", "model.layers.66.mlp.experts.28.up_proj", "model.layers.66.mlp.experts.29.up_proj", "model.layers.66.mlp.experts.30.up_proj", "model.layers.66.mlp.experts.31.up_proj", "model.layers.66.mlp.experts.32.up_proj", "model.layers.66.mlp.experts.33.up_proj", "model.layers.66.mlp.experts.34.up_proj", "model.layers.66.mlp.experts.35.up_proj", "model.layers.66.mlp.experts.36.up_proj", "model.layers.66.mlp.experts.37.up_proj", "model.layers.66.mlp.experts.38.up_proj", "model.layers.66.mlp.experts.39.up_proj", "model.layers.66.mlp.experts.40.up_proj", "model.layers.66.mlp.experts.41.up_proj", "model.layers.66.mlp.experts.42.up_proj", "model.layers.66.mlp.experts.43.up_proj", "model.layers.66.mlp.experts.44.up_proj", "model.layers.66.mlp.experts.45.up_proj", "model.layers.66.mlp.experts.46.up_proj", "model.layers.66.mlp.experts.47.up_proj", "model.layers.66.mlp.experts.48.up_proj", "model.layers.66.mlp.experts.49.up_proj", "model.layers.66.mlp.experts.50.up_proj", "model.layers.66.mlp.experts.51.up_proj", "model.layers.66.mlp.experts.52.up_proj", "model.layers.66.mlp.experts.53.up_proj", "model.layers.66.mlp.experts.54.up_proj", "model.layers.66.mlp.experts.55.up_proj", "model.layers.66.mlp.experts.56.up_proj", "model.layers.66.mlp.experts.57.up_proj", "model.layers.66.mlp.experts.58.up_proj", "model.layers.66.mlp.experts.59.up_proj", "model.layers.66.mlp.experts.60.up_proj", "model.layers.66.mlp.experts.61.up_proj", "model.layers.66.mlp.experts.62.up_proj", "model.layers.66.mlp.experts.63.up_proj", "model.layers.66.mlp.experts.64.up_proj", "model.layers.66.mlp.experts.65.up_proj", "model.layers.66.mlp.experts.66.up_proj", "model.layers.66.mlp.experts.67.up_proj", "model.layers.66.mlp.experts.68.up_proj", "model.layers.66.mlp.experts.69.up_proj", "model.layers.66.mlp.experts.70.up_proj", "model.layers.66.mlp.experts.71.up_proj", "model.layers.66.mlp.experts.72.up_proj", "model.layers.66.mlp.experts.73.up_proj", "model.layers.66.mlp.experts.74.up_proj", "model.layers.66.mlp.experts.75.up_proj", "model.layers.66.mlp.experts.76.up_proj", "model.layers.66.mlp.experts.77.up_proj", "model.layers.66.mlp.experts.78.up_proj", "model.layers.66.mlp.experts.79.up_proj", "model.layers.66.mlp.experts.80.up_proj", "model.layers.66.mlp.experts.81.up_proj", "model.layers.66.mlp.experts.82.up_proj", "model.layers.66.mlp.experts.83.up_proj", "model.layers.66.mlp.experts.84.up_proj", "model.layers.66.mlp.experts.85.up_proj", "model.layers.66.mlp.experts.86.up_proj", "model.layers.66.mlp.experts.87.up_proj", "model.layers.66.mlp.experts.88.up_proj", "model.layers.66.mlp.experts.89.up_proj", "model.layers.66.mlp.experts.90.up_proj", "model.layers.66.mlp.experts.91.up_proj", "model.layers.66.mlp.experts.92.up_proj", "model.layers.66.mlp.experts.93.up_proj", "model.layers.66.mlp.experts.94.up_proj", "model.layers.66.mlp.experts.95.up_proj", "model.layers.66.mlp.experts.96.up_proj", "model.layers.66.mlp.experts.97.up_proj", "model.layers.66.mlp.experts.98.up_proj", "model.layers.66.mlp.experts.99.up_proj", "model.layers.66.mlp.experts.100.up_proj", "model.layers.66.mlp.experts.101.up_proj", "model.layers.66.mlp.experts.102.up_proj", "model.layers.66.mlp.experts.103.up_proj", "model.layers.66.mlp.experts.104.up_proj", "model.layers.66.mlp.experts.105.up_proj", "model.layers.66.mlp.experts.106.up_proj", "model.layers.66.mlp.experts.107.up_proj", "model.layers.66.mlp.experts.108.up_proj", "model.layers.66.mlp.experts.109.up_proj", "model.layers.66.mlp.experts.110.up_proj", "model.layers.66.mlp.experts.111.up_proj", "model.layers.66.mlp.experts.112.up_proj", "model.layers.66.mlp.experts.113.up_proj", "model.layers.66.mlp.experts.114.up_proj", "model.layers.66.mlp.experts.115.up_proj", "model.layers.66.mlp.experts.116.up_proj", "model.layers.66.mlp.experts.117.up_proj", "model.layers.66.mlp.experts.118.up_proj", "model.layers.66.mlp.experts.119.up_proj", "model.layers.66.mlp.experts.120.up_proj", "model.layers.66.mlp.experts.121.up_proj", "model.layers.66.mlp.experts.122.up_proj", "model.layers.66.mlp.experts.123.up_proj", "model.layers.66.mlp.experts.124.up_proj", "model.layers.66.mlp.experts.125.up_proj", "model.layers.66.mlp.experts.126.up_proj", "model.layers.66.mlp.experts.127.up_proj", "model.layers.66.mlp.experts.128.up_proj", "model.layers.66.mlp.experts.129.up_proj", "model.layers.66.mlp.experts.130.up_proj", "model.layers.66.mlp.experts.131.up_proj", "model.layers.66.mlp.experts.132.up_proj", "model.layers.66.mlp.experts.133.up_proj", "model.layers.66.mlp.experts.134.up_proj", "model.layers.66.mlp.experts.135.up_proj", "model.layers.66.mlp.experts.136.up_proj", "model.layers.66.mlp.experts.137.up_proj", "model.layers.66.mlp.experts.138.up_proj", "model.layers.66.mlp.experts.139.up_proj", "model.layers.66.mlp.experts.140.up_proj", "model.layers.66.mlp.experts.141.up_proj", "model.layers.66.mlp.experts.142.up_proj", "model.layers.66.mlp.experts.143.up_proj", "model.layers.66.mlp.experts.144.up_proj", "model.layers.66.mlp.experts.145.up_proj", "model.layers.66.mlp.experts.146.up_proj", "model.layers.66.mlp.experts.147.up_proj", "model.layers.66.mlp.experts.148.up_proj", "model.layers.66.mlp.experts.149.up_proj", "model.layers.66.mlp.experts.150.up_proj", "model.layers.66.mlp.experts.151.up_proj", "model.layers.66.mlp.experts.152.up_proj", "model.layers.66.mlp.experts.153.up_proj", "model.layers.66.mlp.experts.154.up_proj", "model.layers.66.mlp.experts.155.up_proj", "model.layers.66.mlp.experts.156.up_proj", "model.layers.66.mlp.experts.157.up_proj", "model.layers.66.mlp.experts.158.up_proj", "model.layers.66.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 6.075756391510324e-06, "dbits": 5033164800 } ] }, { "idx": 398, "layers": [ "model.layers.66.mlp.experts.0.down_proj", "model.layers.66.mlp.experts.1.down_proj", "model.layers.66.mlp.experts.2.down_proj", "model.layers.66.mlp.experts.3.down_proj", "model.layers.66.mlp.experts.4.down_proj", "model.layers.66.mlp.experts.5.down_proj", "model.layers.66.mlp.experts.6.down_proj", "model.layers.66.mlp.experts.7.down_proj", "model.layers.66.mlp.experts.8.down_proj", "model.layers.66.mlp.experts.9.down_proj", "model.layers.66.mlp.experts.10.down_proj", "model.layers.66.mlp.experts.11.down_proj", "model.layers.66.mlp.experts.12.down_proj", "model.layers.66.mlp.experts.13.down_proj", "model.layers.66.mlp.experts.14.down_proj", "model.layers.66.mlp.experts.15.down_proj", "model.layers.66.mlp.experts.16.down_proj", "model.layers.66.mlp.experts.17.down_proj", "model.layers.66.mlp.experts.18.down_proj", "model.layers.66.mlp.experts.19.down_proj", "model.layers.66.mlp.experts.20.down_proj", "model.layers.66.mlp.experts.21.down_proj", "model.layers.66.mlp.experts.22.down_proj", "model.layers.66.mlp.experts.23.down_proj", "model.layers.66.mlp.experts.24.down_proj", "model.layers.66.mlp.experts.25.down_proj", "model.layers.66.mlp.experts.26.down_proj", "model.layers.66.mlp.experts.27.down_proj", "model.layers.66.mlp.experts.28.down_proj", "model.layers.66.mlp.experts.29.down_proj", "model.layers.66.mlp.experts.30.down_proj", "model.layers.66.mlp.experts.31.down_proj", "model.layers.66.mlp.experts.32.down_proj", "model.layers.66.mlp.experts.33.down_proj", "model.layers.66.mlp.experts.34.down_proj", "model.layers.66.mlp.experts.35.down_proj", "model.layers.66.mlp.experts.36.down_proj", "model.layers.66.mlp.experts.37.down_proj", "model.layers.66.mlp.experts.38.down_proj", "model.layers.66.mlp.experts.39.down_proj", "model.layers.66.mlp.experts.40.down_proj", "model.layers.66.mlp.experts.41.down_proj", "model.layers.66.mlp.experts.42.down_proj", "model.layers.66.mlp.experts.43.down_proj", "model.layers.66.mlp.experts.44.down_proj", "model.layers.66.mlp.experts.45.down_proj", "model.layers.66.mlp.experts.46.down_proj", "model.layers.66.mlp.experts.47.down_proj", "model.layers.66.mlp.experts.48.down_proj", "model.layers.66.mlp.experts.49.down_proj", "model.layers.66.mlp.experts.50.down_proj", "model.layers.66.mlp.experts.51.down_proj", "model.layers.66.mlp.experts.52.down_proj", "model.layers.66.mlp.experts.53.down_proj", "model.layers.66.mlp.experts.54.down_proj", "model.layers.66.mlp.experts.55.down_proj", "model.layers.66.mlp.experts.56.down_proj", "model.layers.66.mlp.experts.57.down_proj", "model.layers.66.mlp.experts.58.down_proj", "model.layers.66.mlp.experts.59.down_proj", "model.layers.66.mlp.experts.60.down_proj", "model.layers.66.mlp.experts.61.down_proj", "model.layers.66.mlp.experts.62.down_proj", "model.layers.66.mlp.experts.63.down_proj", "model.layers.66.mlp.experts.64.down_proj", "model.layers.66.mlp.experts.65.down_proj", "model.layers.66.mlp.experts.66.down_proj", "model.layers.66.mlp.experts.67.down_proj", "model.layers.66.mlp.experts.68.down_proj", "model.layers.66.mlp.experts.69.down_proj", "model.layers.66.mlp.experts.70.down_proj", "model.layers.66.mlp.experts.71.down_proj", "model.layers.66.mlp.experts.72.down_proj", "model.layers.66.mlp.experts.73.down_proj", "model.layers.66.mlp.experts.74.down_proj", "model.layers.66.mlp.experts.75.down_proj", "model.layers.66.mlp.experts.76.down_proj", "model.layers.66.mlp.experts.77.down_proj", "model.layers.66.mlp.experts.78.down_proj", "model.layers.66.mlp.experts.79.down_proj", "model.layers.66.mlp.experts.80.down_proj", "model.layers.66.mlp.experts.81.down_proj", "model.layers.66.mlp.experts.82.down_proj", "model.layers.66.mlp.experts.83.down_proj", "model.layers.66.mlp.experts.84.down_proj", "model.layers.66.mlp.experts.85.down_proj", "model.layers.66.mlp.experts.86.down_proj", "model.layers.66.mlp.experts.87.down_proj", "model.layers.66.mlp.experts.88.down_proj", "model.layers.66.mlp.experts.89.down_proj", "model.layers.66.mlp.experts.90.down_proj", "model.layers.66.mlp.experts.91.down_proj", "model.layers.66.mlp.experts.92.down_proj", "model.layers.66.mlp.experts.93.down_proj", "model.layers.66.mlp.experts.94.down_proj", "model.layers.66.mlp.experts.95.down_proj", "model.layers.66.mlp.experts.96.down_proj", "model.layers.66.mlp.experts.97.down_proj", "model.layers.66.mlp.experts.98.down_proj", "model.layers.66.mlp.experts.99.down_proj", "model.layers.66.mlp.experts.100.down_proj", "model.layers.66.mlp.experts.101.down_proj", "model.layers.66.mlp.experts.102.down_proj", "model.layers.66.mlp.experts.103.down_proj", "model.layers.66.mlp.experts.104.down_proj", "model.layers.66.mlp.experts.105.down_proj", "model.layers.66.mlp.experts.106.down_proj", "model.layers.66.mlp.experts.107.down_proj", "model.layers.66.mlp.experts.108.down_proj", "model.layers.66.mlp.experts.109.down_proj", "model.layers.66.mlp.experts.110.down_proj", "model.layers.66.mlp.experts.111.down_proj", "model.layers.66.mlp.experts.112.down_proj", "model.layers.66.mlp.experts.113.down_proj", "model.layers.66.mlp.experts.114.down_proj", "model.layers.66.mlp.experts.115.down_proj", "model.layers.66.mlp.experts.116.down_proj", "model.layers.66.mlp.experts.117.down_proj", "model.layers.66.mlp.experts.118.down_proj", "model.layers.66.mlp.experts.119.down_proj", "model.layers.66.mlp.experts.120.down_proj", "model.layers.66.mlp.experts.121.down_proj", "model.layers.66.mlp.experts.122.down_proj", "model.layers.66.mlp.experts.123.down_proj", "model.layers.66.mlp.experts.124.down_proj", "model.layers.66.mlp.experts.125.down_proj", "model.layers.66.mlp.experts.126.down_proj", "model.layers.66.mlp.experts.127.down_proj", "model.layers.66.mlp.experts.128.down_proj", "model.layers.66.mlp.experts.129.down_proj", "model.layers.66.mlp.experts.130.down_proj", "model.layers.66.mlp.experts.131.down_proj", "model.layers.66.mlp.experts.132.down_proj", "model.layers.66.mlp.experts.133.down_proj", "model.layers.66.mlp.experts.134.down_proj", "model.layers.66.mlp.experts.135.down_proj", "model.layers.66.mlp.experts.136.down_proj", "model.layers.66.mlp.experts.137.down_proj", "model.layers.66.mlp.experts.138.down_proj", "model.layers.66.mlp.experts.139.down_proj", "model.layers.66.mlp.experts.140.down_proj", "model.layers.66.mlp.experts.141.down_proj", "model.layers.66.mlp.experts.142.down_proj", "model.layers.66.mlp.experts.143.down_proj", "model.layers.66.mlp.experts.144.down_proj", "model.layers.66.mlp.experts.145.down_proj", "model.layers.66.mlp.experts.146.down_proj", "model.layers.66.mlp.experts.147.down_proj", "model.layers.66.mlp.experts.148.down_proj", "model.layers.66.mlp.experts.149.down_proj", "model.layers.66.mlp.experts.150.down_proj", "model.layers.66.mlp.experts.151.down_proj", "model.layers.66.mlp.experts.152.down_proj", "model.layers.66.mlp.experts.153.down_proj", "model.layers.66.mlp.experts.154.down_proj", "model.layers.66.mlp.experts.155.down_proj", "model.layers.66.mlp.experts.156.down_proj", "model.layers.66.mlp.experts.157.down_proj", "model.layers.66.mlp.experts.158.down_proj", "model.layers.66.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.652690793387592e-06, "dbits": 2516582400 } ] }, { "idx": 399, "layers": [ "model.layers.67.self_attn.q_proj" ], "candidates": [ { "dkld": 4.9501861212775545e-06, "dbits": 125829120 } ] }, { "idx": 400, "layers": [ "model.layers.67.self_attn.k_proj", "model.layers.67.self_attn.v_proj" ], "candidates": [ { "dkld": 3.322021802887342e-06, "dbits": 20971520 } ] }, { "idx": 401, "layers": [ "model.layers.67.self_attn.o_proj" ], "candidates": [ { "dkld": 2.567842602729754e-06, "dbits": 125829120 } ] }, { "idx": 402, "layers": [ "model.layers.67.mlp.shared_experts.gate_proj", "model.layers.67.mlp.shared_experts.up_proj", "model.layers.67.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 5.080259870737748e-06, "dbits": 47185920 } ] }, { "idx": 403, "layers": [ "model.layers.67.mlp.experts.0.gate_proj", "model.layers.67.mlp.experts.1.gate_proj", "model.layers.67.mlp.experts.2.gate_proj", "model.layers.67.mlp.experts.3.gate_proj", "model.layers.67.mlp.experts.4.gate_proj", "model.layers.67.mlp.experts.5.gate_proj", "model.layers.67.mlp.experts.6.gate_proj", "model.layers.67.mlp.experts.7.gate_proj", "model.layers.67.mlp.experts.8.gate_proj", "model.layers.67.mlp.experts.9.gate_proj", "model.layers.67.mlp.experts.10.gate_proj", "model.layers.67.mlp.experts.11.gate_proj", "model.layers.67.mlp.experts.12.gate_proj", "model.layers.67.mlp.experts.13.gate_proj", "model.layers.67.mlp.experts.14.gate_proj", "model.layers.67.mlp.experts.15.gate_proj", "model.layers.67.mlp.experts.16.gate_proj", "model.layers.67.mlp.experts.17.gate_proj", "model.layers.67.mlp.experts.18.gate_proj", "model.layers.67.mlp.experts.19.gate_proj", "model.layers.67.mlp.experts.20.gate_proj", "model.layers.67.mlp.experts.21.gate_proj", "model.layers.67.mlp.experts.22.gate_proj", "model.layers.67.mlp.experts.23.gate_proj", "model.layers.67.mlp.experts.24.gate_proj", "model.layers.67.mlp.experts.25.gate_proj", "model.layers.67.mlp.experts.26.gate_proj", "model.layers.67.mlp.experts.27.gate_proj", "model.layers.67.mlp.experts.28.gate_proj", "model.layers.67.mlp.experts.29.gate_proj", "model.layers.67.mlp.experts.30.gate_proj", "model.layers.67.mlp.experts.31.gate_proj", "model.layers.67.mlp.experts.32.gate_proj", "model.layers.67.mlp.experts.33.gate_proj", "model.layers.67.mlp.experts.34.gate_proj", "model.layers.67.mlp.experts.35.gate_proj", "model.layers.67.mlp.experts.36.gate_proj", "model.layers.67.mlp.experts.37.gate_proj", "model.layers.67.mlp.experts.38.gate_proj", "model.layers.67.mlp.experts.39.gate_proj", "model.layers.67.mlp.experts.40.gate_proj", "model.layers.67.mlp.experts.41.gate_proj", "model.layers.67.mlp.experts.42.gate_proj", "model.layers.67.mlp.experts.43.gate_proj", "model.layers.67.mlp.experts.44.gate_proj", "model.layers.67.mlp.experts.45.gate_proj", "model.layers.67.mlp.experts.46.gate_proj", "model.layers.67.mlp.experts.47.gate_proj", "model.layers.67.mlp.experts.48.gate_proj", "model.layers.67.mlp.experts.49.gate_proj", "model.layers.67.mlp.experts.50.gate_proj", "model.layers.67.mlp.experts.51.gate_proj", "model.layers.67.mlp.experts.52.gate_proj", "model.layers.67.mlp.experts.53.gate_proj", "model.layers.67.mlp.experts.54.gate_proj", "model.layers.67.mlp.experts.55.gate_proj", "model.layers.67.mlp.experts.56.gate_proj", "model.layers.67.mlp.experts.57.gate_proj", "model.layers.67.mlp.experts.58.gate_proj", "model.layers.67.mlp.experts.59.gate_proj", "model.layers.67.mlp.experts.60.gate_proj", "model.layers.67.mlp.experts.61.gate_proj", "model.layers.67.mlp.experts.62.gate_proj", "model.layers.67.mlp.experts.63.gate_proj", "model.layers.67.mlp.experts.64.gate_proj", "model.layers.67.mlp.experts.65.gate_proj", "model.layers.67.mlp.experts.66.gate_proj", "model.layers.67.mlp.experts.67.gate_proj", "model.layers.67.mlp.experts.68.gate_proj", "model.layers.67.mlp.experts.69.gate_proj", "model.layers.67.mlp.experts.70.gate_proj", "model.layers.67.mlp.experts.71.gate_proj", "model.layers.67.mlp.experts.72.gate_proj", "model.layers.67.mlp.experts.73.gate_proj", "model.layers.67.mlp.experts.74.gate_proj", "model.layers.67.mlp.experts.75.gate_proj", "model.layers.67.mlp.experts.76.gate_proj", "model.layers.67.mlp.experts.77.gate_proj", "model.layers.67.mlp.experts.78.gate_proj", "model.layers.67.mlp.experts.79.gate_proj", "model.layers.67.mlp.experts.80.gate_proj", "model.layers.67.mlp.experts.81.gate_proj", "model.layers.67.mlp.experts.82.gate_proj", "model.layers.67.mlp.experts.83.gate_proj", "model.layers.67.mlp.experts.84.gate_proj", "model.layers.67.mlp.experts.85.gate_proj", "model.layers.67.mlp.experts.86.gate_proj", "model.layers.67.mlp.experts.87.gate_proj", "model.layers.67.mlp.experts.88.gate_proj", "model.layers.67.mlp.experts.89.gate_proj", "model.layers.67.mlp.experts.90.gate_proj", "model.layers.67.mlp.experts.91.gate_proj", "model.layers.67.mlp.experts.92.gate_proj", "model.layers.67.mlp.experts.93.gate_proj", "model.layers.67.mlp.experts.94.gate_proj", "model.layers.67.mlp.experts.95.gate_proj", "model.layers.67.mlp.experts.96.gate_proj", "model.layers.67.mlp.experts.97.gate_proj", "model.layers.67.mlp.experts.98.gate_proj", "model.layers.67.mlp.experts.99.gate_proj", "model.layers.67.mlp.experts.100.gate_proj", "model.layers.67.mlp.experts.101.gate_proj", "model.layers.67.mlp.experts.102.gate_proj", "model.layers.67.mlp.experts.103.gate_proj", "model.layers.67.mlp.experts.104.gate_proj", "model.layers.67.mlp.experts.105.gate_proj", "model.layers.67.mlp.experts.106.gate_proj", "model.layers.67.mlp.experts.107.gate_proj", "model.layers.67.mlp.experts.108.gate_proj", "model.layers.67.mlp.experts.109.gate_proj", "model.layers.67.mlp.experts.110.gate_proj", "model.layers.67.mlp.experts.111.gate_proj", "model.layers.67.mlp.experts.112.gate_proj", "model.layers.67.mlp.experts.113.gate_proj", "model.layers.67.mlp.experts.114.gate_proj", "model.layers.67.mlp.experts.115.gate_proj", "model.layers.67.mlp.experts.116.gate_proj", "model.layers.67.mlp.experts.117.gate_proj", "model.layers.67.mlp.experts.118.gate_proj", "model.layers.67.mlp.experts.119.gate_proj", "model.layers.67.mlp.experts.120.gate_proj", "model.layers.67.mlp.experts.121.gate_proj", "model.layers.67.mlp.experts.122.gate_proj", "model.layers.67.mlp.experts.123.gate_proj", "model.layers.67.mlp.experts.124.gate_proj", "model.layers.67.mlp.experts.125.gate_proj", "model.layers.67.mlp.experts.126.gate_proj", "model.layers.67.mlp.experts.127.gate_proj", "model.layers.67.mlp.experts.128.gate_proj", "model.layers.67.mlp.experts.129.gate_proj", "model.layers.67.mlp.experts.130.gate_proj", "model.layers.67.mlp.experts.131.gate_proj", "model.layers.67.mlp.experts.132.gate_proj", "model.layers.67.mlp.experts.133.gate_proj", "model.layers.67.mlp.experts.134.gate_proj", "model.layers.67.mlp.experts.135.gate_proj", "model.layers.67.mlp.experts.136.gate_proj", "model.layers.67.mlp.experts.137.gate_proj", "model.layers.67.mlp.experts.138.gate_proj", "model.layers.67.mlp.experts.139.gate_proj", "model.layers.67.mlp.experts.140.gate_proj", "model.layers.67.mlp.experts.141.gate_proj", "model.layers.67.mlp.experts.142.gate_proj", "model.layers.67.mlp.experts.143.gate_proj", "model.layers.67.mlp.experts.144.gate_proj", "model.layers.67.mlp.experts.145.gate_proj", "model.layers.67.mlp.experts.146.gate_proj", "model.layers.67.mlp.experts.147.gate_proj", "model.layers.67.mlp.experts.148.gate_proj", "model.layers.67.mlp.experts.149.gate_proj", "model.layers.67.mlp.experts.150.gate_proj", "model.layers.67.mlp.experts.151.gate_proj", "model.layers.67.mlp.experts.152.gate_proj", "model.layers.67.mlp.experts.153.gate_proj", "model.layers.67.mlp.experts.154.gate_proj", "model.layers.67.mlp.experts.155.gate_proj", "model.layers.67.mlp.experts.156.gate_proj", "model.layers.67.mlp.experts.157.gate_proj", "model.layers.67.mlp.experts.158.gate_proj", "model.layers.67.mlp.experts.159.gate_proj", "model.layers.67.mlp.experts.0.up_proj", "model.layers.67.mlp.experts.1.up_proj", "model.layers.67.mlp.experts.2.up_proj", "model.layers.67.mlp.experts.3.up_proj", "model.layers.67.mlp.experts.4.up_proj", "model.layers.67.mlp.experts.5.up_proj", "model.layers.67.mlp.experts.6.up_proj", "model.layers.67.mlp.experts.7.up_proj", "model.layers.67.mlp.experts.8.up_proj", "model.layers.67.mlp.experts.9.up_proj", "model.layers.67.mlp.experts.10.up_proj", "model.layers.67.mlp.experts.11.up_proj", "model.layers.67.mlp.experts.12.up_proj", "model.layers.67.mlp.experts.13.up_proj", "model.layers.67.mlp.experts.14.up_proj", "model.layers.67.mlp.experts.15.up_proj", "model.layers.67.mlp.experts.16.up_proj", "model.layers.67.mlp.experts.17.up_proj", "model.layers.67.mlp.experts.18.up_proj", "model.layers.67.mlp.experts.19.up_proj", "model.layers.67.mlp.experts.20.up_proj", "model.layers.67.mlp.experts.21.up_proj", "model.layers.67.mlp.experts.22.up_proj", "model.layers.67.mlp.experts.23.up_proj", "model.layers.67.mlp.experts.24.up_proj", "model.layers.67.mlp.experts.25.up_proj", "model.layers.67.mlp.experts.26.up_proj", "model.layers.67.mlp.experts.27.up_proj", "model.layers.67.mlp.experts.28.up_proj", "model.layers.67.mlp.experts.29.up_proj", "model.layers.67.mlp.experts.30.up_proj", "model.layers.67.mlp.experts.31.up_proj", "model.layers.67.mlp.experts.32.up_proj", "model.layers.67.mlp.experts.33.up_proj", "model.layers.67.mlp.experts.34.up_proj", "model.layers.67.mlp.experts.35.up_proj", "model.layers.67.mlp.experts.36.up_proj", "model.layers.67.mlp.experts.37.up_proj", "model.layers.67.mlp.experts.38.up_proj", "model.layers.67.mlp.experts.39.up_proj", "model.layers.67.mlp.experts.40.up_proj", "model.layers.67.mlp.experts.41.up_proj", "model.layers.67.mlp.experts.42.up_proj", "model.layers.67.mlp.experts.43.up_proj", "model.layers.67.mlp.experts.44.up_proj", "model.layers.67.mlp.experts.45.up_proj", "model.layers.67.mlp.experts.46.up_proj", "model.layers.67.mlp.experts.47.up_proj", "model.layers.67.mlp.experts.48.up_proj", "model.layers.67.mlp.experts.49.up_proj", "model.layers.67.mlp.experts.50.up_proj", "model.layers.67.mlp.experts.51.up_proj", "model.layers.67.mlp.experts.52.up_proj", "model.layers.67.mlp.experts.53.up_proj", "model.layers.67.mlp.experts.54.up_proj", "model.layers.67.mlp.experts.55.up_proj", "model.layers.67.mlp.experts.56.up_proj", "model.layers.67.mlp.experts.57.up_proj", "model.layers.67.mlp.experts.58.up_proj", "model.layers.67.mlp.experts.59.up_proj", "model.layers.67.mlp.experts.60.up_proj", "model.layers.67.mlp.experts.61.up_proj", "model.layers.67.mlp.experts.62.up_proj", "model.layers.67.mlp.experts.63.up_proj", "model.layers.67.mlp.experts.64.up_proj", "model.layers.67.mlp.experts.65.up_proj", "model.layers.67.mlp.experts.66.up_proj", "model.layers.67.mlp.experts.67.up_proj", "model.layers.67.mlp.experts.68.up_proj", "model.layers.67.mlp.experts.69.up_proj", "model.layers.67.mlp.experts.70.up_proj", "model.layers.67.mlp.experts.71.up_proj", "model.layers.67.mlp.experts.72.up_proj", "model.layers.67.mlp.experts.73.up_proj", "model.layers.67.mlp.experts.74.up_proj", "model.layers.67.mlp.experts.75.up_proj", "model.layers.67.mlp.experts.76.up_proj", "model.layers.67.mlp.experts.77.up_proj", "model.layers.67.mlp.experts.78.up_proj", "model.layers.67.mlp.experts.79.up_proj", "model.layers.67.mlp.experts.80.up_proj", "model.layers.67.mlp.experts.81.up_proj", "model.layers.67.mlp.experts.82.up_proj", "model.layers.67.mlp.experts.83.up_proj", "model.layers.67.mlp.experts.84.up_proj", "model.layers.67.mlp.experts.85.up_proj", "model.layers.67.mlp.experts.86.up_proj", "model.layers.67.mlp.experts.87.up_proj", "model.layers.67.mlp.experts.88.up_proj", "model.layers.67.mlp.experts.89.up_proj", "model.layers.67.mlp.experts.90.up_proj", "model.layers.67.mlp.experts.91.up_proj", "model.layers.67.mlp.experts.92.up_proj", "model.layers.67.mlp.experts.93.up_proj", "model.layers.67.mlp.experts.94.up_proj", "model.layers.67.mlp.experts.95.up_proj", "model.layers.67.mlp.experts.96.up_proj", "model.layers.67.mlp.experts.97.up_proj", "model.layers.67.mlp.experts.98.up_proj", "model.layers.67.mlp.experts.99.up_proj", "model.layers.67.mlp.experts.100.up_proj", "model.layers.67.mlp.experts.101.up_proj", "model.layers.67.mlp.experts.102.up_proj", "model.layers.67.mlp.experts.103.up_proj", "model.layers.67.mlp.experts.104.up_proj", "model.layers.67.mlp.experts.105.up_proj", "model.layers.67.mlp.experts.106.up_proj", "model.layers.67.mlp.experts.107.up_proj", "model.layers.67.mlp.experts.108.up_proj", "model.layers.67.mlp.experts.109.up_proj", "model.layers.67.mlp.experts.110.up_proj", "model.layers.67.mlp.experts.111.up_proj", "model.layers.67.mlp.experts.112.up_proj", "model.layers.67.mlp.experts.113.up_proj", "model.layers.67.mlp.experts.114.up_proj", "model.layers.67.mlp.experts.115.up_proj", "model.layers.67.mlp.experts.116.up_proj", "model.layers.67.mlp.experts.117.up_proj", "model.layers.67.mlp.experts.118.up_proj", "model.layers.67.mlp.experts.119.up_proj", "model.layers.67.mlp.experts.120.up_proj", "model.layers.67.mlp.experts.121.up_proj", "model.layers.67.mlp.experts.122.up_proj", "model.layers.67.mlp.experts.123.up_proj", "model.layers.67.mlp.experts.124.up_proj", "model.layers.67.mlp.experts.125.up_proj", "model.layers.67.mlp.experts.126.up_proj", "model.layers.67.mlp.experts.127.up_proj", "model.layers.67.mlp.experts.128.up_proj", "model.layers.67.mlp.experts.129.up_proj", "model.layers.67.mlp.experts.130.up_proj", "model.layers.67.mlp.experts.131.up_proj", "model.layers.67.mlp.experts.132.up_proj", "model.layers.67.mlp.experts.133.up_proj", "model.layers.67.mlp.experts.134.up_proj", "model.layers.67.mlp.experts.135.up_proj", "model.layers.67.mlp.experts.136.up_proj", "model.layers.67.mlp.experts.137.up_proj", "model.layers.67.mlp.experts.138.up_proj", "model.layers.67.mlp.experts.139.up_proj", "model.layers.67.mlp.experts.140.up_proj", "model.layers.67.mlp.experts.141.up_proj", "model.layers.67.mlp.experts.142.up_proj", "model.layers.67.mlp.experts.143.up_proj", "model.layers.67.mlp.experts.144.up_proj", "model.layers.67.mlp.experts.145.up_proj", "model.layers.67.mlp.experts.146.up_proj", "model.layers.67.mlp.experts.147.up_proj", "model.layers.67.mlp.experts.148.up_proj", "model.layers.67.mlp.experts.149.up_proj", "model.layers.67.mlp.experts.150.up_proj", "model.layers.67.mlp.experts.151.up_proj", "model.layers.67.mlp.experts.152.up_proj", "model.layers.67.mlp.experts.153.up_proj", "model.layers.67.mlp.experts.154.up_proj", "model.layers.67.mlp.experts.155.up_proj", "model.layers.67.mlp.experts.156.up_proj", "model.layers.67.mlp.experts.157.up_proj", "model.layers.67.mlp.experts.158.up_proj", "model.layers.67.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 4.01775760110472e-06, "dbits": 5033164800 } ] }, { "idx": 404, "layers": [ "model.layers.67.mlp.experts.0.down_proj", "model.layers.67.mlp.experts.1.down_proj", "model.layers.67.mlp.experts.2.down_proj", "model.layers.67.mlp.experts.3.down_proj", "model.layers.67.mlp.experts.4.down_proj", "model.layers.67.mlp.experts.5.down_proj", "model.layers.67.mlp.experts.6.down_proj", "model.layers.67.mlp.experts.7.down_proj", "model.layers.67.mlp.experts.8.down_proj", "model.layers.67.mlp.experts.9.down_proj", "model.layers.67.mlp.experts.10.down_proj", "model.layers.67.mlp.experts.11.down_proj", "model.layers.67.mlp.experts.12.down_proj", "model.layers.67.mlp.experts.13.down_proj", "model.layers.67.mlp.experts.14.down_proj", "model.layers.67.mlp.experts.15.down_proj", "model.layers.67.mlp.experts.16.down_proj", "model.layers.67.mlp.experts.17.down_proj", "model.layers.67.mlp.experts.18.down_proj", "model.layers.67.mlp.experts.19.down_proj", "model.layers.67.mlp.experts.20.down_proj", "model.layers.67.mlp.experts.21.down_proj", "model.layers.67.mlp.experts.22.down_proj", "model.layers.67.mlp.experts.23.down_proj", "model.layers.67.mlp.experts.24.down_proj", "model.layers.67.mlp.experts.25.down_proj", "model.layers.67.mlp.experts.26.down_proj", "model.layers.67.mlp.experts.27.down_proj", "model.layers.67.mlp.experts.28.down_proj", "model.layers.67.mlp.experts.29.down_proj", "model.layers.67.mlp.experts.30.down_proj", "model.layers.67.mlp.experts.31.down_proj", "model.layers.67.mlp.experts.32.down_proj", "model.layers.67.mlp.experts.33.down_proj", "model.layers.67.mlp.experts.34.down_proj", "model.layers.67.mlp.experts.35.down_proj", "model.layers.67.mlp.experts.36.down_proj", "model.layers.67.mlp.experts.37.down_proj", "model.layers.67.mlp.experts.38.down_proj", "model.layers.67.mlp.experts.39.down_proj", "model.layers.67.mlp.experts.40.down_proj", "model.layers.67.mlp.experts.41.down_proj", "model.layers.67.mlp.experts.42.down_proj", "model.layers.67.mlp.experts.43.down_proj", "model.layers.67.mlp.experts.44.down_proj", "model.layers.67.mlp.experts.45.down_proj", "model.layers.67.mlp.experts.46.down_proj", "model.layers.67.mlp.experts.47.down_proj", "model.layers.67.mlp.experts.48.down_proj", "model.layers.67.mlp.experts.49.down_proj", "model.layers.67.mlp.experts.50.down_proj", "model.layers.67.mlp.experts.51.down_proj", "model.layers.67.mlp.experts.52.down_proj", "model.layers.67.mlp.experts.53.down_proj", "model.layers.67.mlp.experts.54.down_proj", "model.layers.67.mlp.experts.55.down_proj", "model.layers.67.mlp.experts.56.down_proj", "model.layers.67.mlp.experts.57.down_proj", "model.layers.67.mlp.experts.58.down_proj", "model.layers.67.mlp.experts.59.down_proj", "model.layers.67.mlp.experts.60.down_proj", "model.layers.67.mlp.experts.61.down_proj", "model.layers.67.mlp.experts.62.down_proj", "model.layers.67.mlp.experts.63.down_proj", "model.layers.67.mlp.experts.64.down_proj", "model.layers.67.mlp.experts.65.down_proj", "model.layers.67.mlp.experts.66.down_proj", "model.layers.67.mlp.experts.67.down_proj", "model.layers.67.mlp.experts.68.down_proj", "model.layers.67.mlp.experts.69.down_proj", "model.layers.67.mlp.experts.70.down_proj", "model.layers.67.mlp.experts.71.down_proj", "model.layers.67.mlp.experts.72.down_proj", "model.layers.67.mlp.experts.73.down_proj", "model.layers.67.mlp.experts.74.down_proj", "model.layers.67.mlp.experts.75.down_proj", "model.layers.67.mlp.experts.76.down_proj", "model.layers.67.mlp.experts.77.down_proj", "model.layers.67.mlp.experts.78.down_proj", "model.layers.67.mlp.experts.79.down_proj", "model.layers.67.mlp.experts.80.down_proj", "model.layers.67.mlp.experts.81.down_proj", "model.layers.67.mlp.experts.82.down_proj", "model.layers.67.mlp.experts.83.down_proj", "model.layers.67.mlp.experts.84.down_proj", "model.layers.67.mlp.experts.85.down_proj", "model.layers.67.mlp.experts.86.down_proj", "model.layers.67.mlp.experts.87.down_proj", "model.layers.67.mlp.experts.88.down_proj", "model.layers.67.mlp.experts.89.down_proj", "model.layers.67.mlp.experts.90.down_proj", "model.layers.67.mlp.experts.91.down_proj", "model.layers.67.mlp.experts.92.down_proj", "model.layers.67.mlp.experts.93.down_proj", "model.layers.67.mlp.experts.94.down_proj", "model.layers.67.mlp.experts.95.down_proj", "model.layers.67.mlp.experts.96.down_proj", "model.layers.67.mlp.experts.97.down_proj", "model.layers.67.mlp.experts.98.down_proj", "model.layers.67.mlp.experts.99.down_proj", "model.layers.67.mlp.experts.100.down_proj", "model.layers.67.mlp.experts.101.down_proj", "model.layers.67.mlp.experts.102.down_proj", "model.layers.67.mlp.experts.103.down_proj", "model.layers.67.mlp.experts.104.down_proj", "model.layers.67.mlp.experts.105.down_proj", "model.layers.67.mlp.experts.106.down_proj", "model.layers.67.mlp.experts.107.down_proj", "model.layers.67.mlp.experts.108.down_proj", "model.layers.67.mlp.experts.109.down_proj", "model.layers.67.mlp.experts.110.down_proj", "model.layers.67.mlp.experts.111.down_proj", "model.layers.67.mlp.experts.112.down_proj", "model.layers.67.mlp.experts.113.down_proj", "model.layers.67.mlp.experts.114.down_proj", "model.layers.67.mlp.experts.115.down_proj", "model.layers.67.mlp.experts.116.down_proj", "model.layers.67.mlp.experts.117.down_proj", "model.layers.67.mlp.experts.118.down_proj", "model.layers.67.mlp.experts.119.down_proj", "model.layers.67.mlp.experts.120.down_proj", "model.layers.67.mlp.experts.121.down_proj", "model.layers.67.mlp.experts.122.down_proj", "model.layers.67.mlp.experts.123.down_proj", "model.layers.67.mlp.experts.124.down_proj", "model.layers.67.mlp.experts.125.down_proj", "model.layers.67.mlp.experts.126.down_proj", "model.layers.67.mlp.experts.127.down_proj", "model.layers.67.mlp.experts.128.down_proj", "model.layers.67.mlp.experts.129.down_proj", "model.layers.67.mlp.experts.130.down_proj", "model.layers.67.mlp.experts.131.down_proj", "model.layers.67.mlp.experts.132.down_proj", "model.layers.67.mlp.experts.133.down_proj", "model.layers.67.mlp.experts.134.down_proj", "model.layers.67.mlp.experts.135.down_proj", "model.layers.67.mlp.experts.136.down_proj", "model.layers.67.mlp.experts.137.down_proj", "model.layers.67.mlp.experts.138.down_proj", "model.layers.67.mlp.experts.139.down_proj", "model.layers.67.mlp.experts.140.down_proj", "model.layers.67.mlp.experts.141.down_proj", "model.layers.67.mlp.experts.142.down_proj", "model.layers.67.mlp.experts.143.down_proj", "model.layers.67.mlp.experts.144.down_proj", "model.layers.67.mlp.experts.145.down_proj", "model.layers.67.mlp.experts.146.down_proj", "model.layers.67.mlp.experts.147.down_proj", "model.layers.67.mlp.experts.148.down_proj", "model.layers.67.mlp.experts.149.down_proj", "model.layers.67.mlp.experts.150.down_proj", "model.layers.67.mlp.experts.151.down_proj", "model.layers.67.mlp.experts.152.down_proj", "model.layers.67.mlp.experts.153.down_proj", "model.layers.67.mlp.experts.154.down_proj", "model.layers.67.mlp.experts.155.down_proj", "model.layers.67.mlp.experts.156.down_proj", "model.layers.67.mlp.experts.157.down_proj", "model.layers.67.mlp.experts.158.down_proj", "model.layers.67.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 7.060103234834882e-06, "dbits": 2516582400 } ] }, { "idx": 405, "layers": [ "model.layers.68.self_attn.q_proj" ], "candidates": [ { "dkld": 1.2162781786173582e-06, "dbits": 125829120 } ] }, { "idx": 406, "layers": [ "model.layers.68.self_attn.k_proj", "model.layers.68.self_attn.v_proj" ], "candidates": [ { "dkld": -9.858049452302672e-08, "dbits": 20971520 } ] }, { "idx": 407, "layers": [ "model.layers.68.self_attn.o_proj" ], "candidates": [ { "dkld": -1.2583128409460404e-06, "dbits": 125829120 } ] }, { "idx": 408, "layers": [ "model.layers.68.mlp.shared_experts.gate_proj", "model.layers.68.mlp.shared_experts.up_proj", "model.layers.68.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.175901995040433e-06, "dbits": 47185920 } ] }, { "idx": 409, "layers": [ "model.layers.68.mlp.experts.0.gate_proj", "model.layers.68.mlp.experts.1.gate_proj", "model.layers.68.mlp.experts.2.gate_proj", "model.layers.68.mlp.experts.3.gate_proj", "model.layers.68.mlp.experts.4.gate_proj", "model.layers.68.mlp.experts.5.gate_proj", "model.layers.68.mlp.experts.6.gate_proj", "model.layers.68.mlp.experts.7.gate_proj", "model.layers.68.mlp.experts.8.gate_proj", "model.layers.68.mlp.experts.9.gate_proj", "model.layers.68.mlp.experts.10.gate_proj", "model.layers.68.mlp.experts.11.gate_proj", "model.layers.68.mlp.experts.12.gate_proj", "model.layers.68.mlp.experts.13.gate_proj", "model.layers.68.mlp.experts.14.gate_proj", "model.layers.68.mlp.experts.15.gate_proj", "model.layers.68.mlp.experts.16.gate_proj", "model.layers.68.mlp.experts.17.gate_proj", "model.layers.68.mlp.experts.18.gate_proj", "model.layers.68.mlp.experts.19.gate_proj", "model.layers.68.mlp.experts.20.gate_proj", "model.layers.68.mlp.experts.21.gate_proj", "model.layers.68.mlp.experts.22.gate_proj", "model.layers.68.mlp.experts.23.gate_proj", "model.layers.68.mlp.experts.24.gate_proj", "model.layers.68.mlp.experts.25.gate_proj", "model.layers.68.mlp.experts.26.gate_proj", "model.layers.68.mlp.experts.27.gate_proj", "model.layers.68.mlp.experts.28.gate_proj", "model.layers.68.mlp.experts.29.gate_proj", "model.layers.68.mlp.experts.30.gate_proj", "model.layers.68.mlp.experts.31.gate_proj", "model.layers.68.mlp.experts.32.gate_proj", "model.layers.68.mlp.experts.33.gate_proj", "model.layers.68.mlp.experts.34.gate_proj", "model.layers.68.mlp.experts.35.gate_proj", "model.layers.68.mlp.experts.36.gate_proj", "model.layers.68.mlp.experts.37.gate_proj", "model.layers.68.mlp.experts.38.gate_proj", "model.layers.68.mlp.experts.39.gate_proj", "model.layers.68.mlp.experts.40.gate_proj", "model.layers.68.mlp.experts.41.gate_proj", "model.layers.68.mlp.experts.42.gate_proj", "model.layers.68.mlp.experts.43.gate_proj", "model.layers.68.mlp.experts.44.gate_proj", "model.layers.68.mlp.experts.45.gate_proj", "model.layers.68.mlp.experts.46.gate_proj", "model.layers.68.mlp.experts.47.gate_proj", "model.layers.68.mlp.experts.48.gate_proj", "model.layers.68.mlp.experts.49.gate_proj", "model.layers.68.mlp.experts.50.gate_proj", "model.layers.68.mlp.experts.51.gate_proj", "model.layers.68.mlp.experts.52.gate_proj", "model.layers.68.mlp.experts.53.gate_proj", "model.layers.68.mlp.experts.54.gate_proj", "model.layers.68.mlp.experts.55.gate_proj", "model.layers.68.mlp.experts.56.gate_proj", "model.layers.68.mlp.experts.57.gate_proj", "model.layers.68.mlp.experts.58.gate_proj", "model.layers.68.mlp.experts.59.gate_proj", "model.layers.68.mlp.experts.60.gate_proj", "model.layers.68.mlp.experts.61.gate_proj", "model.layers.68.mlp.experts.62.gate_proj", "model.layers.68.mlp.experts.63.gate_proj", "model.layers.68.mlp.experts.64.gate_proj", "model.layers.68.mlp.experts.65.gate_proj", "model.layers.68.mlp.experts.66.gate_proj", "model.layers.68.mlp.experts.67.gate_proj", "model.layers.68.mlp.experts.68.gate_proj", "model.layers.68.mlp.experts.69.gate_proj", "model.layers.68.mlp.experts.70.gate_proj", "model.layers.68.mlp.experts.71.gate_proj", "model.layers.68.mlp.experts.72.gate_proj", "model.layers.68.mlp.experts.73.gate_proj", "model.layers.68.mlp.experts.74.gate_proj", "model.layers.68.mlp.experts.75.gate_proj", "model.layers.68.mlp.experts.76.gate_proj", "model.layers.68.mlp.experts.77.gate_proj", "model.layers.68.mlp.experts.78.gate_proj", "model.layers.68.mlp.experts.79.gate_proj", "model.layers.68.mlp.experts.80.gate_proj", "model.layers.68.mlp.experts.81.gate_proj", "model.layers.68.mlp.experts.82.gate_proj", "model.layers.68.mlp.experts.83.gate_proj", "model.layers.68.mlp.experts.84.gate_proj", "model.layers.68.mlp.experts.85.gate_proj", "model.layers.68.mlp.experts.86.gate_proj", "model.layers.68.mlp.experts.87.gate_proj", "model.layers.68.mlp.experts.88.gate_proj", "model.layers.68.mlp.experts.89.gate_proj", "model.layers.68.mlp.experts.90.gate_proj", "model.layers.68.mlp.experts.91.gate_proj", "model.layers.68.mlp.experts.92.gate_proj", "model.layers.68.mlp.experts.93.gate_proj", "model.layers.68.mlp.experts.94.gate_proj", "model.layers.68.mlp.experts.95.gate_proj", "model.layers.68.mlp.experts.96.gate_proj", "model.layers.68.mlp.experts.97.gate_proj", "model.layers.68.mlp.experts.98.gate_proj", "model.layers.68.mlp.experts.99.gate_proj", "model.layers.68.mlp.experts.100.gate_proj", "model.layers.68.mlp.experts.101.gate_proj", "model.layers.68.mlp.experts.102.gate_proj", "model.layers.68.mlp.experts.103.gate_proj", "model.layers.68.mlp.experts.104.gate_proj", "model.layers.68.mlp.experts.105.gate_proj", "model.layers.68.mlp.experts.106.gate_proj", "model.layers.68.mlp.experts.107.gate_proj", "model.layers.68.mlp.experts.108.gate_proj", "model.layers.68.mlp.experts.109.gate_proj", "model.layers.68.mlp.experts.110.gate_proj", "model.layers.68.mlp.experts.111.gate_proj", "model.layers.68.mlp.experts.112.gate_proj", "model.layers.68.mlp.experts.113.gate_proj", "model.layers.68.mlp.experts.114.gate_proj", "model.layers.68.mlp.experts.115.gate_proj", "model.layers.68.mlp.experts.116.gate_proj", "model.layers.68.mlp.experts.117.gate_proj", "model.layers.68.mlp.experts.118.gate_proj", "model.layers.68.mlp.experts.119.gate_proj", "model.layers.68.mlp.experts.120.gate_proj", "model.layers.68.mlp.experts.121.gate_proj", "model.layers.68.mlp.experts.122.gate_proj", "model.layers.68.mlp.experts.123.gate_proj", "model.layers.68.mlp.experts.124.gate_proj", "model.layers.68.mlp.experts.125.gate_proj", "model.layers.68.mlp.experts.126.gate_proj", "model.layers.68.mlp.experts.127.gate_proj", "model.layers.68.mlp.experts.128.gate_proj", "model.layers.68.mlp.experts.129.gate_proj", "model.layers.68.mlp.experts.130.gate_proj", "model.layers.68.mlp.experts.131.gate_proj", "model.layers.68.mlp.experts.132.gate_proj", "model.layers.68.mlp.experts.133.gate_proj", "model.layers.68.mlp.experts.134.gate_proj", "model.layers.68.mlp.experts.135.gate_proj", "model.layers.68.mlp.experts.136.gate_proj", "model.layers.68.mlp.experts.137.gate_proj", "model.layers.68.mlp.experts.138.gate_proj", "model.layers.68.mlp.experts.139.gate_proj", "model.layers.68.mlp.experts.140.gate_proj", "model.layers.68.mlp.experts.141.gate_proj", "model.layers.68.mlp.experts.142.gate_proj", "model.layers.68.mlp.experts.143.gate_proj", "model.layers.68.mlp.experts.144.gate_proj", "model.layers.68.mlp.experts.145.gate_proj", "model.layers.68.mlp.experts.146.gate_proj", "model.layers.68.mlp.experts.147.gate_proj", "model.layers.68.mlp.experts.148.gate_proj", "model.layers.68.mlp.experts.149.gate_proj", "model.layers.68.mlp.experts.150.gate_proj", "model.layers.68.mlp.experts.151.gate_proj", "model.layers.68.mlp.experts.152.gate_proj", "model.layers.68.mlp.experts.153.gate_proj", "model.layers.68.mlp.experts.154.gate_proj", "model.layers.68.mlp.experts.155.gate_proj", "model.layers.68.mlp.experts.156.gate_proj", "model.layers.68.mlp.experts.157.gate_proj", "model.layers.68.mlp.experts.158.gate_proj", "model.layers.68.mlp.experts.159.gate_proj", "model.layers.68.mlp.experts.0.up_proj", "model.layers.68.mlp.experts.1.up_proj", "model.layers.68.mlp.experts.2.up_proj", "model.layers.68.mlp.experts.3.up_proj", "model.layers.68.mlp.experts.4.up_proj", "model.layers.68.mlp.experts.5.up_proj", "model.layers.68.mlp.experts.6.up_proj", "model.layers.68.mlp.experts.7.up_proj", "model.layers.68.mlp.experts.8.up_proj", "model.layers.68.mlp.experts.9.up_proj", "model.layers.68.mlp.experts.10.up_proj", "model.layers.68.mlp.experts.11.up_proj", "model.layers.68.mlp.experts.12.up_proj", "model.layers.68.mlp.experts.13.up_proj", "model.layers.68.mlp.experts.14.up_proj", "model.layers.68.mlp.experts.15.up_proj", "model.layers.68.mlp.experts.16.up_proj", "model.layers.68.mlp.experts.17.up_proj", "model.layers.68.mlp.experts.18.up_proj", "model.layers.68.mlp.experts.19.up_proj", "model.layers.68.mlp.experts.20.up_proj", "model.layers.68.mlp.experts.21.up_proj", "model.layers.68.mlp.experts.22.up_proj", "model.layers.68.mlp.experts.23.up_proj", "model.layers.68.mlp.experts.24.up_proj", "model.layers.68.mlp.experts.25.up_proj", "model.layers.68.mlp.experts.26.up_proj", "model.layers.68.mlp.experts.27.up_proj", "model.layers.68.mlp.experts.28.up_proj", "model.layers.68.mlp.experts.29.up_proj", "model.layers.68.mlp.experts.30.up_proj", "model.layers.68.mlp.experts.31.up_proj", "model.layers.68.mlp.experts.32.up_proj", "model.layers.68.mlp.experts.33.up_proj", "model.layers.68.mlp.experts.34.up_proj", "model.layers.68.mlp.experts.35.up_proj", "model.layers.68.mlp.experts.36.up_proj", "model.layers.68.mlp.experts.37.up_proj", "model.layers.68.mlp.experts.38.up_proj", "model.layers.68.mlp.experts.39.up_proj", "model.layers.68.mlp.experts.40.up_proj", "model.layers.68.mlp.experts.41.up_proj", "model.layers.68.mlp.experts.42.up_proj", "model.layers.68.mlp.experts.43.up_proj", "model.layers.68.mlp.experts.44.up_proj", "model.layers.68.mlp.experts.45.up_proj", "model.layers.68.mlp.experts.46.up_proj", "model.layers.68.mlp.experts.47.up_proj", "model.layers.68.mlp.experts.48.up_proj", "model.layers.68.mlp.experts.49.up_proj", "model.layers.68.mlp.experts.50.up_proj", "model.layers.68.mlp.experts.51.up_proj", "model.layers.68.mlp.experts.52.up_proj", "model.layers.68.mlp.experts.53.up_proj", "model.layers.68.mlp.experts.54.up_proj", "model.layers.68.mlp.experts.55.up_proj", "model.layers.68.mlp.experts.56.up_proj", "model.layers.68.mlp.experts.57.up_proj", "model.layers.68.mlp.experts.58.up_proj", "model.layers.68.mlp.experts.59.up_proj", "model.layers.68.mlp.experts.60.up_proj", "model.layers.68.mlp.experts.61.up_proj", "model.layers.68.mlp.experts.62.up_proj", "model.layers.68.mlp.experts.63.up_proj", "model.layers.68.mlp.experts.64.up_proj", "model.layers.68.mlp.experts.65.up_proj", "model.layers.68.mlp.experts.66.up_proj", "model.layers.68.mlp.experts.67.up_proj", "model.layers.68.mlp.experts.68.up_proj", "model.layers.68.mlp.experts.69.up_proj", "model.layers.68.mlp.experts.70.up_proj", "model.layers.68.mlp.experts.71.up_proj", "model.layers.68.mlp.experts.72.up_proj", "model.layers.68.mlp.experts.73.up_proj", "model.layers.68.mlp.experts.74.up_proj", "model.layers.68.mlp.experts.75.up_proj", "model.layers.68.mlp.experts.76.up_proj", "model.layers.68.mlp.experts.77.up_proj", "model.layers.68.mlp.experts.78.up_proj", "model.layers.68.mlp.experts.79.up_proj", "model.layers.68.mlp.experts.80.up_proj", "model.layers.68.mlp.experts.81.up_proj", "model.layers.68.mlp.experts.82.up_proj", "model.layers.68.mlp.experts.83.up_proj", "model.layers.68.mlp.experts.84.up_proj", "model.layers.68.mlp.experts.85.up_proj", "model.layers.68.mlp.experts.86.up_proj", "model.layers.68.mlp.experts.87.up_proj", "model.layers.68.mlp.experts.88.up_proj", "model.layers.68.mlp.experts.89.up_proj", "model.layers.68.mlp.experts.90.up_proj", "model.layers.68.mlp.experts.91.up_proj", "model.layers.68.mlp.experts.92.up_proj", "model.layers.68.mlp.experts.93.up_proj", "model.layers.68.mlp.experts.94.up_proj", "model.layers.68.mlp.experts.95.up_proj", "model.layers.68.mlp.experts.96.up_proj", "model.layers.68.mlp.experts.97.up_proj", "model.layers.68.mlp.experts.98.up_proj", "model.layers.68.mlp.experts.99.up_proj", "model.layers.68.mlp.experts.100.up_proj", "model.layers.68.mlp.experts.101.up_proj", "model.layers.68.mlp.experts.102.up_proj", "model.layers.68.mlp.experts.103.up_proj", "model.layers.68.mlp.experts.104.up_proj", "model.layers.68.mlp.experts.105.up_proj", "model.layers.68.mlp.experts.106.up_proj", "model.layers.68.mlp.experts.107.up_proj", "model.layers.68.mlp.experts.108.up_proj", "model.layers.68.mlp.experts.109.up_proj", "model.layers.68.mlp.experts.110.up_proj", "model.layers.68.mlp.experts.111.up_proj", "model.layers.68.mlp.experts.112.up_proj", "model.layers.68.mlp.experts.113.up_proj", "model.layers.68.mlp.experts.114.up_proj", "model.layers.68.mlp.experts.115.up_proj", "model.layers.68.mlp.experts.116.up_proj", "model.layers.68.mlp.experts.117.up_proj", "model.layers.68.mlp.experts.118.up_proj", "model.layers.68.mlp.experts.119.up_proj", "model.layers.68.mlp.experts.120.up_proj", "model.layers.68.mlp.experts.121.up_proj", "model.layers.68.mlp.experts.122.up_proj", "model.layers.68.mlp.experts.123.up_proj", "model.layers.68.mlp.experts.124.up_proj", "model.layers.68.mlp.experts.125.up_proj", "model.layers.68.mlp.experts.126.up_proj", "model.layers.68.mlp.experts.127.up_proj", "model.layers.68.mlp.experts.128.up_proj", "model.layers.68.mlp.experts.129.up_proj", "model.layers.68.mlp.experts.130.up_proj", "model.layers.68.mlp.experts.131.up_proj", "model.layers.68.mlp.experts.132.up_proj", "model.layers.68.mlp.experts.133.up_proj", "model.layers.68.mlp.experts.134.up_proj", "model.layers.68.mlp.experts.135.up_proj", "model.layers.68.mlp.experts.136.up_proj", "model.layers.68.mlp.experts.137.up_proj", "model.layers.68.mlp.experts.138.up_proj", "model.layers.68.mlp.experts.139.up_proj", "model.layers.68.mlp.experts.140.up_proj", "model.layers.68.mlp.experts.141.up_proj", "model.layers.68.mlp.experts.142.up_proj", "model.layers.68.mlp.experts.143.up_proj", "model.layers.68.mlp.experts.144.up_proj", "model.layers.68.mlp.experts.145.up_proj", "model.layers.68.mlp.experts.146.up_proj", "model.layers.68.mlp.experts.147.up_proj", "model.layers.68.mlp.experts.148.up_proj", "model.layers.68.mlp.experts.149.up_proj", "model.layers.68.mlp.experts.150.up_proj", "model.layers.68.mlp.experts.151.up_proj", "model.layers.68.mlp.experts.152.up_proj", "model.layers.68.mlp.experts.153.up_proj", "model.layers.68.mlp.experts.154.up_proj", "model.layers.68.mlp.experts.155.up_proj", "model.layers.68.mlp.experts.156.up_proj", "model.layers.68.mlp.experts.157.up_proj", "model.layers.68.mlp.experts.158.up_proj", "model.layers.68.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.6640278520062783e-06, "dbits": 5033164800 } ] }, { "idx": 410, "layers": [ "model.layers.68.mlp.experts.0.down_proj", "model.layers.68.mlp.experts.1.down_proj", "model.layers.68.mlp.experts.2.down_proj", "model.layers.68.mlp.experts.3.down_proj", "model.layers.68.mlp.experts.4.down_proj", "model.layers.68.mlp.experts.5.down_proj", "model.layers.68.mlp.experts.6.down_proj", "model.layers.68.mlp.experts.7.down_proj", "model.layers.68.mlp.experts.8.down_proj", "model.layers.68.mlp.experts.9.down_proj", "model.layers.68.mlp.experts.10.down_proj", "model.layers.68.mlp.experts.11.down_proj", "model.layers.68.mlp.experts.12.down_proj", "model.layers.68.mlp.experts.13.down_proj", "model.layers.68.mlp.experts.14.down_proj", "model.layers.68.mlp.experts.15.down_proj", "model.layers.68.mlp.experts.16.down_proj", "model.layers.68.mlp.experts.17.down_proj", "model.layers.68.mlp.experts.18.down_proj", "model.layers.68.mlp.experts.19.down_proj", "model.layers.68.mlp.experts.20.down_proj", "model.layers.68.mlp.experts.21.down_proj", "model.layers.68.mlp.experts.22.down_proj", "model.layers.68.mlp.experts.23.down_proj", "model.layers.68.mlp.experts.24.down_proj", "model.layers.68.mlp.experts.25.down_proj", "model.layers.68.mlp.experts.26.down_proj", "model.layers.68.mlp.experts.27.down_proj", "model.layers.68.mlp.experts.28.down_proj", "model.layers.68.mlp.experts.29.down_proj", "model.layers.68.mlp.experts.30.down_proj", "model.layers.68.mlp.experts.31.down_proj", "model.layers.68.mlp.experts.32.down_proj", "model.layers.68.mlp.experts.33.down_proj", "model.layers.68.mlp.experts.34.down_proj", "model.layers.68.mlp.experts.35.down_proj", "model.layers.68.mlp.experts.36.down_proj", "model.layers.68.mlp.experts.37.down_proj", "model.layers.68.mlp.experts.38.down_proj", "model.layers.68.mlp.experts.39.down_proj", "model.layers.68.mlp.experts.40.down_proj", "model.layers.68.mlp.experts.41.down_proj", "model.layers.68.mlp.experts.42.down_proj", "model.layers.68.mlp.experts.43.down_proj", "model.layers.68.mlp.experts.44.down_proj", "model.layers.68.mlp.experts.45.down_proj", "model.layers.68.mlp.experts.46.down_proj", "model.layers.68.mlp.experts.47.down_proj", "model.layers.68.mlp.experts.48.down_proj", "model.layers.68.mlp.experts.49.down_proj", "model.layers.68.mlp.experts.50.down_proj", "model.layers.68.mlp.experts.51.down_proj", "model.layers.68.mlp.experts.52.down_proj", "model.layers.68.mlp.experts.53.down_proj", "model.layers.68.mlp.experts.54.down_proj", "model.layers.68.mlp.experts.55.down_proj", "model.layers.68.mlp.experts.56.down_proj", "model.layers.68.mlp.experts.57.down_proj", "model.layers.68.mlp.experts.58.down_proj", "model.layers.68.mlp.experts.59.down_proj", "model.layers.68.mlp.experts.60.down_proj", "model.layers.68.mlp.experts.61.down_proj", "model.layers.68.mlp.experts.62.down_proj", "model.layers.68.mlp.experts.63.down_proj", "model.layers.68.mlp.experts.64.down_proj", "model.layers.68.mlp.experts.65.down_proj", "model.layers.68.mlp.experts.66.down_proj", "model.layers.68.mlp.experts.67.down_proj", "model.layers.68.mlp.experts.68.down_proj", "model.layers.68.mlp.experts.69.down_proj", "model.layers.68.mlp.experts.70.down_proj", "model.layers.68.mlp.experts.71.down_proj", "model.layers.68.mlp.experts.72.down_proj", "model.layers.68.mlp.experts.73.down_proj", "model.layers.68.mlp.experts.74.down_proj", "model.layers.68.mlp.experts.75.down_proj", "model.layers.68.mlp.experts.76.down_proj", "model.layers.68.mlp.experts.77.down_proj", "model.layers.68.mlp.experts.78.down_proj", "model.layers.68.mlp.experts.79.down_proj", "model.layers.68.mlp.experts.80.down_proj", "model.layers.68.mlp.experts.81.down_proj", "model.layers.68.mlp.experts.82.down_proj", "model.layers.68.mlp.experts.83.down_proj", "model.layers.68.mlp.experts.84.down_proj", "model.layers.68.mlp.experts.85.down_proj", "model.layers.68.mlp.experts.86.down_proj", "model.layers.68.mlp.experts.87.down_proj", "model.layers.68.mlp.experts.88.down_proj", "model.layers.68.mlp.experts.89.down_proj", "model.layers.68.mlp.experts.90.down_proj", "model.layers.68.mlp.experts.91.down_proj", "model.layers.68.mlp.experts.92.down_proj", "model.layers.68.mlp.experts.93.down_proj", "model.layers.68.mlp.experts.94.down_proj", "model.layers.68.mlp.experts.95.down_proj", "model.layers.68.mlp.experts.96.down_proj", "model.layers.68.mlp.experts.97.down_proj", "model.layers.68.mlp.experts.98.down_proj", "model.layers.68.mlp.experts.99.down_proj", "model.layers.68.mlp.experts.100.down_proj", "model.layers.68.mlp.experts.101.down_proj", "model.layers.68.mlp.experts.102.down_proj", "model.layers.68.mlp.experts.103.down_proj", "model.layers.68.mlp.experts.104.down_proj", "model.layers.68.mlp.experts.105.down_proj", "model.layers.68.mlp.experts.106.down_proj", "model.layers.68.mlp.experts.107.down_proj", "model.layers.68.mlp.experts.108.down_proj", "model.layers.68.mlp.experts.109.down_proj", "model.layers.68.mlp.experts.110.down_proj", "model.layers.68.mlp.experts.111.down_proj", "model.layers.68.mlp.experts.112.down_proj", "model.layers.68.mlp.experts.113.down_proj", "model.layers.68.mlp.experts.114.down_proj", "model.layers.68.mlp.experts.115.down_proj", "model.layers.68.mlp.experts.116.down_proj", "model.layers.68.mlp.experts.117.down_proj", "model.layers.68.mlp.experts.118.down_proj", "model.layers.68.mlp.experts.119.down_proj", "model.layers.68.mlp.experts.120.down_proj", "model.layers.68.mlp.experts.121.down_proj", "model.layers.68.mlp.experts.122.down_proj", "model.layers.68.mlp.experts.123.down_proj", "model.layers.68.mlp.experts.124.down_proj", "model.layers.68.mlp.experts.125.down_proj", "model.layers.68.mlp.experts.126.down_proj", "model.layers.68.mlp.experts.127.down_proj", "model.layers.68.mlp.experts.128.down_proj", "model.layers.68.mlp.experts.129.down_proj", "model.layers.68.mlp.experts.130.down_proj", "model.layers.68.mlp.experts.131.down_proj", "model.layers.68.mlp.experts.132.down_proj", "model.layers.68.mlp.experts.133.down_proj", "model.layers.68.mlp.experts.134.down_proj", "model.layers.68.mlp.experts.135.down_proj", "model.layers.68.mlp.experts.136.down_proj", "model.layers.68.mlp.experts.137.down_proj", "model.layers.68.mlp.experts.138.down_proj", "model.layers.68.mlp.experts.139.down_proj", "model.layers.68.mlp.experts.140.down_proj", "model.layers.68.mlp.experts.141.down_proj", "model.layers.68.mlp.experts.142.down_proj", "model.layers.68.mlp.experts.143.down_proj", "model.layers.68.mlp.experts.144.down_proj", "model.layers.68.mlp.experts.145.down_proj", "model.layers.68.mlp.experts.146.down_proj", "model.layers.68.mlp.experts.147.down_proj", "model.layers.68.mlp.experts.148.down_proj", "model.layers.68.mlp.experts.149.down_proj", "model.layers.68.mlp.experts.150.down_proj", "model.layers.68.mlp.experts.151.down_proj", "model.layers.68.mlp.experts.152.down_proj", "model.layers.68.mlp.experts.153.down_proj", "model.layers.68.mlp.experts.154.down_proj", "model.layers.68.mlp.experts.155.down_proj", "model.layers.68.mlp.experts.156.down_proj", "model.layers.68.mlp.experts.157.down_proj", "model.layers.68.mlp.experts.158.down_proj", "model.layers.68.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.163617268204711e-06, "dbits": 2516582400 } ] }, { "idx": 411, "layers": [ "model.layers.69.self_attn.q_proj" ], "candidates": [ { "dkld": 5.956040695309639e-06, "dbits": 125829120 } ] }, { "idx": 412, "layers": [ "model.layers.69.self_attn.k_proj", "model.layers.69.self_attn.v_proj" ], "candidates": [ { "dkld": 3.1887291697784825e-06, "dbits": 20971520 } ] }, { "idx": 413, "layers": [ "model.layers.69.self_attn.o_proj" ], "candidates": [ { "dkld": 5.820655496791092e-06, "dbits": 125829120 } ] }, { "idx": 414, "layers": [ "model.layers.69.mlp.shared_experts.gate_proj", "model.layers.69.mlp.shared_experts.up_proj", "model.layers.69.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.857379826716992e-06, "dbits": 47185920 } ] }, { "idx": 415, "layers": [ "model.layers.69.mlp.experts.0.gate_proj", "model.layers.69.mlp.experts.1.gate_proj", "model.layers.69.mlp.experts.2.gate_proj", "model.layers.69.mlp.experts.3.gate_proj", "model.layers.69.mlp.experts.4.gate_proj", "model.layers.69.mlp.experts.5.gate_proj", "model.layers.69.mlp.experts.6.gate_proj", "model.layers.69.mlp.experts.7.gate_proj", "model.layers.69.mlp.experts.8.gate_proj", "model.layers.69.mlp.experts.9.gate_proj", "model.layers.69.mlp.experts.10.gate_proj", "model.layers.69.mlp.experts.11.gate_proj", "model.layers.69.mlp.experts.12.gate_proj", "model.layers.69.mlp.experts.13.gate_proj", "model.layers.69.mlp.experts.14.gate_proj", "model.layers.69.mlp.experts.15.gate_proj", "model.layers.69.mlp.experts.16.gate_proj", "model.layers.69.mlp.experts.17.gate_proj", "model.layers.69.mlp.experts.18.gate_proj", "model.layers.69.mlp.experts.19.gate_proj", "model.layers.69.mlp.experts.20.gate_proj", "model.layers.69.mlp.experts.21.gate_proj", "model.layers.69.mlp.experts.22.gate_proj", "model.layers.69.mlp.experts.23.gate_proj", "model.layers.69.mlp.experts.24.gate_proj", "model.layers.69.mlp.experts.25.gate_proj", "model.layers.69.mlp.experts.26.gate_proj", "model.layers.69.mlp.experts.27.gate_proj", "model.layers.69.mlp.experts.28.gate_proj", "model.layers.69.mlp.experts.29.gate_proj", "model.layers.69.mlp.experts.30.gate_proj", "model.layers.69.mlp.experts.31.gate_proj", "model.layers.69.mlp.experts.32.gate_proj", "model.layers.69.mlp.experts.33.gate_proj", "model.layers.69.mlp.experts.34.gate_proj", "model.layers.69.mlp.experts.35.gate_proj", "model.layers.69.mlp.experts.36.gate_proj", "model.layers.69.mlp.experts.37.gate_proj", "model.layers.69.mlp.experts.38.gate_proj", "model.layers.69.mlp.experts.39.gate_proj", "model.layers.69.mlp.experts.40.gate_proj", "model.layers.69.mlp.experts.41.gate_proj", "model.layers.69.mlp.experts.42.gate_proj", "model.layers.69.mlp.experts.43.gate_proj", "model.layers.69.mlp.experts.44.gate_proj", "model.layers.69.mlp.experts.45.gate_proj", "model.layers.69.mlp.experts.46.gate_proj", "model.layers.69.mlp.experts.47.gate_proj", "model.layers.69.mlp.experts.48.gate_proj", "model.layers.69.mlp.experts.49.gate_proj", "model.layers.69.mlp.experts.50.gate_proj", "model.layers.69.mlp.experts.51.gate_proj", "model.layers.69.mlp.experts.52.gate_proj", "model.layers.69.mlp.experts.53.gate_proj", "model.layers.69.mlp.experts.54.gate_proj", "model.layers.69.mlp.experts.55.gate_proj", "model.layers.69.mlp.experts.56.gate_proj", "model.layers.69.mlp.experts.57.gate_proj", "model.layers.69.mlp.experts.58.gate_proj", "model.layers.69.mlp.experts.59.gate_proj", "model.layers.69.mlp.experts.60.gate_proj", "model.layers.69.mlp.experts.61.gate_proj", "model.layers.69.mlp.experts.62.gate_proj", "model.layers.69.mlp.experts.63.gate_proj", "model.layers.69.mlp.experts.64.gate_proj", "model.layers.69.mlp.experts.65.gate_proj", "model.layers.69.mlp.experts.66.gate_proj", "model.layers.69.mlp.experts.67.gate_proj", "model.layers.69.mlp.experts.68.gate_proj", "model.layers.69.mlp.experts.69.gate_proj", "model.layers.69.mlp.experts.70.gate_proj", "model.layers.69.mlp.experts.71.gate_proj", "model.layers.69.mlp.experts.72.gate_proj", "model.layers.69.mlp.experts.73.gate_proj", "model.layers.69.mlp.experts.74.gate_proj", "model.layers.69.mlp.experts.75.gate_proj", "model.layers.69.mlp.experts.76.gate_proj", "model.layers.69.mlp.experts.77.gate_proj", "model.layers.69.mlp.experts.78.gate_proj", "model.layers.69.mlp.experts.79.gate_proj", "model.layers.69.mlp.experts.80.gate_proj", "model.layers.69.mlp.experts.81.gate_proj", "model.layers.69.mlp.experts.82.gate_proj", "model.layers.69.mlp.experts.83.gate_proj", "model.layers.69.mlp.experts.84.gate_proj", "model.layers.69.mlp.experts.85.gate_proj", "model.layers.69.mlp.experts.86.gate_proj", "model.layers.69.mlp.experts.87.gate_proj", "model.layers.69.mlp.experts.88.gate_proj", "model.layers.69.mlp.experts.89.gate_proj", "model.layers.69.mlp.experts.90.gate_proj", "model.layers.69.mlp.experts.91.gate_proj", "model.layers.69.mlp.experts.92.gate_proj", "model.layers.69.mlp.experts.93.gate_proj", "model.layers.69.mlp.experts.94.gate_proj", "model.layers.69.mlp.experts.95.gate_proj", "model.layers.69.mlp.experts.96.gate_proj", "model.layers.69.mlp.experts.97.gate_proj", "model.layers.69.mlp.experts.98.gate_proj", "model.layers.69.mlp.experts.99.gate_proj", "model.layers.69.mlp.experts.100.gate_proj", "model.layers.69.mlp.experts.101.gate_proj", "model.layers.69.mlp.experts.102.gate_proj", "model.layers.69.mlp.experts.103.gate_proj", "model.layers.69.mlp.experts.104.gate_proj", "model.layers.69.mlp.experts.105.gate_proj", "model.layers.69.mlp.experts.106.gate_proj", "model.layers.69.mlp.experts.107.gate_proj", "model.layers.69.mlp.experts.108.gate_proj", "model.layers.69.mlp.experts.109.gate_proj", "model.layers.69.mlp.experts.110.gate_proj", "model.layers.69.mlp.experts.111.gate_proj", "model.layers.69.mlp.experts.112.gate_proj", "model.layers.69.mlp.experts.113.gate_proj", "model.layers.69.mlp.experts.114.gate_proj", "model.layers.69.mlp.experts.115.gate_proj", "model.layers.69.mlp.experts.116.gate_proj", "model.layers.69.mlp.experts.117.gate_proj", "model.layers.69.mlp.experts.118.gate_proj", "model.layers.69.mlp.experts.119.gate_proj", "model.layers.69.mlp.experts.120.gate_proj", "model.layers.69.mlp.experts.121.gate_proj", "model.layers.69.mlp.experts.122.gate_proj", "model.layers.69.mlp.experts.123.gate_proj", "model.layers.69.mlp.experts.124.gate_proj", "model.layers.69.mlp.experts.125.gate_proj", "model.layers.69.mlp.experts.126.gate_proj", "model.layers.69.mlp.experts.127.gate_proj", "model.layers.69.mlp.experts.128.gate_proj", "model.layers.69.mlp.experts.129.gate_proj", "model.layers.69.mlp.experts.130.gate_proj", "model.layers.69.mlp.experts.131.gate_proj", "model.layers.69.mlp.experts.132.gate_proj", "model.layers.69.mlp.experts.133.gate_proj", "model.layers.69.mlp.experts.134.gate_proj", "model.layers.69.mlp.experts.135.gate_proj", "model.layers.69.mlp.experts.136.gate_proj", "model.layers.69.mlp.experts.137.gate_proj", "model.layers.69.mlp.experts.138.gate_proj", "model.layers.69.mlp.experts.139.gate_proj", "model.layers.69.mlp.experts.140.gate_proj", "model.layers.69.mlp.experts.141.gate_proj", "model.layers.69.mlp.experts.142.gate_proj", "model.layers.69.mlp.experts.143.gate_proj", "model.layers.69.mlp.experts.144.gate_proj", "model.layers.69.mlp.experts.145.gate_proj", "model.layers.69.mlp.experts.146.gate_proj", "model.layers.69.mlp.experts.147.gate_proj", "model.layers.69.mlp.experts.148.gate_proj", "model.layers.69.mlp.experts.149.gate_proj", "model.layers.69.mlp.experts.150.gate_proj", "model.layers.69.mlp.experts.151.gate_proj", "model.layers.69.mlp.experts.152.gate_proj", "model.layers.69.mlp.experts.153.gate_proj", "model.layers.69.mlp.experts.154.gate_proj", "model.layers.69.mlp.experts.155.gate_proj", "model.layers.69.mlp.experts.156.gate_proj", "model.layers.69.mlp.experts.157.gate_proj", "model.layers.69.mlp.experts.158.gate_proj", "model.layers.69.mlp.experts.159.gate_proj", "model.layers.69.mlp.experts.0.up_proj", "model.layers.69.mlp.experts.1.up_proj", "model.layers.69.mlp.experts.2.up_proj", "model.layers.69.mlp.experts.3.up_proj", "model.layers.69.mlp.experts.4.up_proj", "model.layers.69.mlp.experts.5.up_proj", "model.layers.69.mlp.experts.6.up_proj", "model.layers.69.mlp.experts.7.up_proj", "model.layers.69.mlp.experts.8.up_proj", "model.layers.69.mlp.experts.9.up_proj", "model.layers.69.mlp.experts.10.up_proj", "model.layers.69.mlp.experts.11.up_proj", "model.layers.69.mlp.experts.12.up_proj", "model.layers.69.mlp.experts.13.up_proj", "model.layers.69.mlp.experts.14.up_proj", "model.layers.69.mlp.experts.15.up_proj", "model.layers.69.mlp.experts.16.up_proj", "model.layers.69.mlp.experts.17.up_proj", "model.layers.69.mlp.experts.18.up_proj", "model.layers.69.mlp.experts.19.up_proj", "model.layers.69.mlp.experts.20.up_proj", "model.layers.69.mlp.experts.21.up_proj", "model.layers.69.mlp.experts.22.up_proj", "model.layers.69.mlp.experts.23.up_proj", "model.layers.69.mlp.experts.24.up_proj", "model.layers.69.mlp.experts.25.up_proj", "model.layers.69.mlp.experts.26.up_proj", "model.layers.69.mlp.experts.27.up_proj", "model.layers.69.mlp.experts.28.up_proj", "model.layers.69.mlp.experts.29.up_proj", "model.layers.69.mlp.experts.30.up_proj", "model.layers.69.mlp.experts.31.up_proj", "model.layers.69.mlp.experts.32.up_proj", "model.layers.69.mlp.experts.33.up_proj", "model.layers.69.mlp.experts.34.up_proj", "model.layers.69.mlp.experts.35.up_proj", "model.layers.69.mlp.experts.36.up_proj", "model.layers.69.mlp.experts.37.up_proj", "model.layers.69.mlp.experts.38.up_proj", "model.layers.69.mlp.experts.39.up_proj", "model.layers.69.mlp.experts.40.up_proj", "model.layers.69.mlp.experts.41.up_proj", "model.layers.69.mlp.experts.42.up_proj", "model.layers.69.mlp.experts.43.up_proj", "model.layers.69.mlp.experts.44.up_proj", "model.layers.69.mlp.experts.45.up_proj", "model.layers.69.mlp.experts.46.up_proj", "model.layers.69.mlp.experts.47.up_proj", "model.layers.69.mlp.experts.48.up_proj", "model.layers.69.mlp.experts.49.up_proj", "model.layers.69.mlp.experts.50.up_proj", "model.layers.69.mlp.experts.51.up_proj", "model.layers.69.mlp.experts.52.up_proj", "model.layers.69.mlp.experts.53.up_proj", "model.layers.69.mlp.experts.54.up_proj", "model.layers.69.mlp.experts.55.up_proj", "model.layers.69.mlp.experts.56.up_proj", "model.layers.69.mlp.experts.57.up_proj", "model.layers.69.mlp.experts.58.up_proj", "model.layers.69.mlp.experts.59.up_proj", "model.layers.69.mlp.experts.60.up_proj", "model.layers.69.mlp.experts.61.up_proj", "model.layers.69.mlp.experts.62.up_proj", "model.layers.69.mlp.experts.63.up_proj", "model.layers.69.mlp.experts.64.up_proj", "model.layers.69.mlp.experts.65.up_proj", "model.layers.69.mlp.experts.66.up_proj", "model.layers.69.mlp.experts.67.up_proj", "model.layers.69.mlp.experts.68.up_proj", "model.layers.69.mlp.experts.69.up_proj", "model.layers.69.mlp.experts.70.up_proj", "model.layers.69.mlp.experts.71.up_proj", "model.layers.69.mlp.experts.72.up_proj", "model.layers.69.mlp.experts.73.up_proj", "model.layers.69.mlp.experts.74.up_proj", "model.layers.69.mlp.experts.75.up_proj", "model.layers.69.mlp.experts.76.up_proj", "model.layers.69.mlp.experts.77.up_proj", "model.layers.69.mlp.experts.78.up_proj", "model.layers.69.mlp.experts.79.up_proj", "model.layers.69.mlp.experts.80.up_proj", "model.layers.69.mlp.experts.81.up_proj", "model.layers.69.mlp.experts.82.up_proj", "model.layers.69.mlp.experts.83.up_proj", "model.layers.69.mlp.experts.84.up_proj", "model.layers.69.mlp.experts.85.up_proj", "model.layers.69.mlp.experts.86.up_proj", "model.layers.69.mlp.experts.87.up_proj", "model.layers.69.mlp.experts.88.up_proj", "model.layers.69.mlp.experts.89.up_proj", "model.layers.69.mlp.experts.90.up_proj", "model.layers.69.mlp.experts.91.up_proj", "model.layers.69.mlp.experts.92.up_proj", "model.layers.69.mlp.experts.93.up_proj", "model.layers.69.mlp.experts.94.up_proj", "model.layers.69.mlp.experts.95.up_proj", "model.layers.69.mlp.experts.96.up_proj", "model.layers.69.mlp.experts.97.up_proj", "model.layers.69.mlp.experts.98.up_proj", "model.layers.69.mlp.experts.99.up_proj", "model.layers.69.mlp.experts.100.up_proj", "model.layers.69.mlp.experts.101.up_proj", "model.layers.69.mlp.experts.102.up_proj", "model.layers.69.mlp.experts.103.up_proj", "model.layers.69.mlp.experts.104.up_proj", "model.layers.69.mlp.experts.105.up_proj", "model.layers.69.mlp.experts.106.up_proj", "model.layers.69.mlp.experts.107.up_proj", "model.layers.69.mlp.experts.108.up_proj", "model.layers.69.mlp.experts.109.up_proj", "model.layers.69.mlp.experts.110.up_proj", "model.layers.69.mlp.experts.111.up_proj", "model.layers.69.mlp.experts.112.up_proj", "model.layers.69.mlp.experts.113.up_proj", "model.layers.69.mlp.experts.114.up_proj", "model.layers.69.mlp.experts.115.up_proj", "model.layers.69.mlp.experts.116.up_proj", "model.layers.69.mlp.experts.117.up_proj", "model.layers.69.mlp.experts.118.up_proj", "model.layers.69.mlp.experts.119.up_proj", "model.layers.69.mlp.experts.120.up_proj", "model.layers.69.mlp.experts.121.up_proj", "model.layers.69.mlp.experts.122.up_proj", "model.layers.69.mlp.experts.123.up_proj", "model.layers.69.mlp.experts.124.up_proj", "model.layers.69.mlp.experts.125.up_proj", "model.layers.69.mlp.experts.126.up_proj", "model.layers.69.mlp.experts.127.up_proj", "model.layers.69.mlp.experts.128.up_proj", "model.layers.69.mlp.experts.129.up_proj", "model.layers.69.mlp.experts.130.up_proj", "model.layers.69.mlp.experts.131.up_proj", "model.layers.69.mlp.experts.132.up_proj", "model.layers.69.mlp.experts.133.up_proj", "model.layers.69.mlp.experts.134.up_proj", "model.layers.69.mlp.experts.135.up_proj", "model.layers.69.mlp.experts.136.up_proj", "model.layers.69.mlp.experts.137.up_proj", "model.layers.69.mlp.experts.138.up_proj", "model.layers.69.mlp.experts.139.up_proj", "model.layers.69.mlp.experts.140.up_proj", "model.layers.69.mlp.experts.141.up_proj", "model.layers.69.mlp.experts.142.up_proj", "model.layers.69.mlp.experts.143.up_proj", "model.layers.69.mlp.experts.144.up_proj", "model.layers.69.mlp.experts.145.up_proj", "model.layers.69.mlp.experts.146.up_proj", "model.layers.69.mlp.experts.147.up_proj", "model.layers.69.mlp.experts.148.up_proj", "model.layers.69.mlp.experts.149.up_proj", "model.layers.69.mlp.experts.150.up_proj", "model.layers.69.mlp.experts.151.up_proj", "model.layers.69.mlp.experts.152.up_proj", "model.layers.69.mlp.experts.153.up_proj", "model.layers.69.mlp.experts.154.up_proj", "model.layers.69.mlp.experts.155.up_proj", "model.layers.69.mlp.experts.156.up_proj", "model.layers.69.mlp.experts.157.up_proj", "model.layers.69.mlp.experts.158.up_proj", "model.layers.69.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.8385675502940796e-06, "dbits": 5033164800 } ] }, { "idx": 416, "layers": [ "model.layers.69.mlp.experts.0.down_proj", "model.layers.69.mlp.experts.1.down_proj", "model.layers.69.mlp.experts.2.down_proj", "model.layers.69.mlp.experts.3.down_proj", "model.layers.69.mlp.experts.4.down_proj", "model.layers.69.mlp.experts.5.down_proj", "model.layers.69.mlp.experts.6.down_proj", "model.layers.69.mlp.experts.7.down_proj", "model.layers.69.mlp.experts.8.down_proj", "model.layers.69.mlp.experts.9.down_proj", "model.layers.69.mlp.experts.10.down_proj", "model.layers.69.mlp.experts.11.down_proj", "model.layers.69.mlp.experts.12.down_proj", "model.layers.69.mlp.experts.13.down_proj", "model.layers.69.mlp.experts.14.down_proj", "model.layers.69.mlp.experts.15.down_proj", "model.layers.69.mlp.experts.16.down_proj", "model.layers.69.mlp.experts.17.down_proj", "model.layers.69.mlp.experts.18.down_proj", "model.layers.69.mlp.experts.19.down_proj", "model.layers.69.mlp.experts.20.down_proj", "model.layers.69.mlp.experts.21.down_proj", "model.layers.69.mlp.experts.22.down_proj", "model.layers.69.mlp.experts.23.down_proj", "model.layers.69.mlp.experts.24.down_proj", "model.layers.69.mlp.experts.25.down_proj", "model.layers.69.mlp.experts.26.down_proj", "model.layers.69.mlp.experts.27.down_proj", "model.layers.69.mlp.experts.28.down_proj", "model.layers.69.mlp.experts.29.down_proj", "model.layers.69.mlp.experts.30.down_proj", "model.layers.69.mlp.experts.31.down_proj", "model.layers.69.mlp.experts.32.down_proj", "model.layers.69.mlp.experts.33.down_proj", "model.layers.69.mlp.experts.34.down_proj", "model.layers.69.mlp.experts.35.down_proj", "model.layers.69.mlp.experts.36.down_proj", "model.layers.69.mlp.experts.37.down_proj", "model.layers.69.mlp.experts.38.down_proj", "model.layers.69.mlp.experts.39.down_proj", "model.layers.69.mlp.experts.40.down_proj", "model.layers.69.mlp.experts.41.down_proj", "model.layers.69.mlp.experts.42.down_proj", "model.layers.69.mlp.experts.43.down_proj", "model.layers.69.mlp.experts.44.down_proj", "model.layers.69.mlp.experts.45.down_proj", "model.layers.69.mlp.experts.46.down_proj", "model.layers.69.mlp.experts.47.down_proj", "model.layers.69.mlp.experts.48.down_proj", "model.layers.69.mlp.experts.49.down_proj", "model.layers.69.mlp.experts.50.down_proj", "model.layers.69.mlp.experts.51.down_proj", "model.layers.69.mlp.experts.52.down_proj", "model.layers.69.mlp.experts.53.down_proj", "model.layers.69.mlp.experts.54.down_proj", "model.layers.69.mlp.experts.55.down_proj", "model.layers.69.mlp.experts.56.down_proj", "model.layers.69.mlp.experts.57.down_proj", "model.layers.69.mlp.experts.58.down_proj", "model.layers.69.mlp.experts.59.down_proj", "model.layers.69.mlp.experts.60.down_proj", "model.layers.69.mlp.experts.61.down_proj", "model.layers.69.mlp.experts.62.down_proj", "model.layers.69.mlp.experts.63.down_proj", "model.layers.69.mlp.experts.64.down_proj", "model.layers.69.mlp.experts.65.down_proj", "model.layers.69.mlp.experts.66.down_proj", "model.layers.69.mlp.experts.67.down_proj", "model.layers.69.mlp.experts.68.down_proj", "model.layers.69.mlp.experts.69.down_proj", "model.layers.69.mlp.experts.70.down_proj", "model.layers.69.mlp.experts.71.down_proj", "model.layers.69.mlp.experts.72.down_proj", "model.layers.69.mlp.experts.73.down_proj", "model.layers.69.mlp.experts.74.down_proj", "model.layers.69.mlp.experts.75.down_proj", "model.layers.69.mlp.experts.76.down_proj", "model.layers.69.mlp.experts.77.down_proj", "model.layers.69.mlp.experts.78.down_proj", "model.layers.69.mlp.experts.79.down_proj", "model.layers.69.mlp.experts.80.down_proj", "model.layers.69.mlp.experts.81.down_proj", "model.layers.69.mlp.experts.82.down_proj", "model.layers.69.mlp.experts.83.down_proj", "model.layers.69.mlp.experts.84.down_proj", "model.layers.69.mlp.experts.85.down_proj", "model.layers.69.mlp.experts.86.down_proj", "model.layers.69.mlp.experts.87.down_proj", "model.layers.69.mlp.experts.88.down_proj", "model.layers.69.mlp.experts.89.down_proj", "model.layers.69.mlp.experts.90.down_proj", "model.layers.69.mlp.experts.91.down_proj", "model.layers.69.mlp.experts.92.down_proj", "model.layers.69.mlp.experts.93.down_proj", "model.layers.69.mlp.experts.94.down_proj", "model.layers.69.mlp.experts.95.down_proj", "model.layers.69.mlp.experts.96.down_proj", "model.layers.69.mlp.experts.97.down_proj", "model.layers.69.mlp.experts.98.down_proj", "model.layers.69.mlp.experts.99.down_proj", "model.layers.69.mlp.experts.100.down_proj", "model.layers.69.mlp.experts.101.down_proj", "model.layers.69.mlp.experts.102.down_proj", "model.layers.69.mlp.experts.103.down_proj", "model.layers.69.mlp.experts.104.down_proj", "model.layers.69.mlp.experts.105.down_proj", "model.layers.69.mlp.experts.106.down_proj", "model.layers.69.mlp.experts.107.down_proj", "model.layers.69.mlp.experts.108.down_proj", "model.layers.69.mlp.experts.109.down_proj", "model.layers.69.mlp.experts.110.down_proj", "model.layers.69.mlp.experts.111.down_proj", "model.layers.69.mlp.experts.112.down_proj", "model.layers.69.mlp.experts.113.down_proj", "model.layers.69.mlp.experts.114.down_proj", "model.layers.69.mlp.experts.115.down_proj", "model.layers.69.mlp.experts.116.down_proj", "model.layers.69.mlp.experts.117.down_proj", "model.layers.69.mlp.experts.118.down_proj", "model.layers.69.mlp.experts.119.down_proj", "model.layers.69.mlp.experts.120.down_proj", "model.layers.69.mlp.experts.121.down_proj", "model.layers.69.mlp.experts.122.down_proj", "model.layers.69.mlp.experts.123.down_proj", "model.layers.69.mlp.experts.124.down_proj", "model.layers.69.mlp.experts.125.down_proj", "model.layers.69.mlp.experts.126.down_proj", "model.layers.69.mlp.experts.127.down_proj", "model.layers.69.mlp.experts.128.down_proj", "model.layers.69.mlp.experts.129.down_proj", "model.layers.69.mlp.experts.130.down_proj", "model.layers.69.mlp.experts.131.down_proj", "model.layers.69.mlp.experts.132.down_proj", "model.layers.69.mlp.experts.133.down_proj", "model.layers.69.mlp.experts.134.down_proj", "model.layers.69.mlp.experts.135.down_proj", "model.layers.69.mlp.experts.136.down_proj", "model.layers.69.mlp.experts.137.down_proj", "model.layers.69.mlp.experts.138.down_proj", "model.layers.69.mlp.experts.139.down_proj", "model.layers.69.mlp.experts.140.down_proj", "model.layers.69.mlp.experts.141.down_proj", "model.layers.69.mlp.experts.142.down_proj", "model.layers.69.mlp.experts.143.down_proj", "model.layers.69.mlp.experts.144.down_proj", "model.layers.69.mlp.experts.145.down_proj", "model.layers.69.mlp.experts.146.down_proj", "model.layers.69.mlp.experts.147.down_proj", "model.layers.69.mlp.experts.148.down_proj", "model.layers.69.mlp.experts.149.down_proj", "model.layers.69.mlp.experts.150.down_proj", "model.layers.69.mlp.experts.151.down_proj", "model.layers.69.mlp.experts.152.down_proj", "model.layers.69.mlp.experts.153.down_proj", "model.layers.69.mlp.experts.154.down_proj", "model.layers.69.mlp.experts.155.down_proj", "model.layers.69.mlp.experts.156.down_proj", "model.layers.69.mlp.experts.157.down_proj", "model.layers.69.mlp.experts.158.down_proj", "model.layers.69.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.5638676965608325e-06, "dbits": 2516582400 } ] }, { "idx": 417, "layers": [ "model.layers.70.self_attn.q_proj" ], "candidates": [ { "dkld": 4.19587304349993e-06, "dbits": 125829120 } ] }, { "idx": 418, "layers": [ "model.layers.70.self_attn.k_proj", "model.layers.70.self_attn.v_proj" ], "candidates": [ { "dkld": 4.316540434956637e-06, "dbits": 20971520 } ] }, { "idx": 419, "layers": [ "model.layers.70.self_attn.o_proj" ], "candidates": [ { "dkld": 3.099499735981226e-06, "dbits": 125829120 } ] }, { "idx": 420, "layers": [ "model.layers.70.mlp.shared_experts.gate_proj", "model.layers.70.mlp.shared_experts.up_proj", "model.layers.70.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.3333570677787282e-06, "dbits": 47185920 } ] }, { "idx": 421, "layers": [ "model.layers.70.mlp.experts.0.gate_proj", "model.layers.70.mlp.experts.1.gate_proj", "model.layers.70.mlp.experts.2.gate_proj", "model.layers.70.mlp.experts.3.gate_proj", "model.layers.70.mlp.experts.4.gate_proj", "model.layers.70.mlp.experts.5.gate_proj", "model.layers.70.mlp.experts.6.gate_proj", "model.layers.70.mlp.experts.7.gate_proj", "model.layers.70.mlp.experts.8.gate_proj", "model.layers.70.mlp.experts.9.gate_proj", "model.layers.70.mlp.experts.10.gate_proj", "model.layers.70.mlp.experts.11.gate_proj", "model.layers.70.mlp.experts.12.gate_proj", "model.layers.70.mlp.experts.13.gate_proj", "model.layers.70.mlp.experts.14.gate_proj", "model.layers.70.mlp.experts.15.gate_proj", "model.layers.70.mlp.experts.16.gate_proj", "model.layers.70.mlp.experts.17.gate_proj", "model.layers.70.mlp.experts.18.gate_proj", "model.layers.70.mlp.experts.19.gate_proj", "model.layers.70.mlp.experts.20.gate_proj", "model.layers.70.mlp.experts.21.gate_proj", "model.layers.70.mlp.experts.22.gate_proj", "model.layers.70.mlp.experts.23.gate_proj", "model.layers.70.mlp.experts.24.gate_proj", "model.layers.70.mlp.experts.25.gate_proj", "model.layers.70.mlp.experts.26.gate_proj", "model.layers.70.mlp.experts.27.gate_proj", "model.layers.70.mlp.experts.28.gate_proj", "model.layers.70.mlp.experts.29.gate_proj", "model.layers.70.mlp.experts.30.gate_proj", "model.layers.70.mlp.experts.31.gate_proj", "model.layers.70.mlp.experts.32.gate_proj", "model.layers.70.mlp.experts.33.gate_proj", "model.layers.70.mlp.experts.34.gate_proj", "model.layers.70.mlp.experts.35.gate_proj", "model.layers.70.mlp.experts.36.gate_proj", "model.layers.70.mlp.experts.37.gate_proj", "model.layers.70.mlp.experts.38.gate_proj", "model.layers.70.mlp.experts.39.gate_proj", "model.layers.70.mlp.experts.40.gate_proj", "model.layers.70.mlp.experts.41.gate_proj", "model.layers.70.mlp.experts.42.gate_proj", "model.layers.70.mlp.experts.43.gate_proj", "model.layers.70.mlp.experts.44.gate_proj", "model.layers.70.mlp.experts.45.gate_proj", "model.layers.70.mlp.experts.46.gate_proj", "model.layers.70.mlp.experts.47.gate_proj", "model.layers.70.mlp.experts.48.gate_proj", "model.layers.70.mlp.experts.49.gate_proj", "model.layers.70.mlp.experts.50.gate_proj", "model.layers.70.mlp.experts.51.gate_proj", "model.layers.70.mlp.experts.52.gate_proj", "model.layers.70.mlp.experts.53.gate_proj", "model.layers.70.mlp.experts.54.gate_proj", "model.layers.70.mlp.experts.55.gate_proj", "model.layers.70.mlp.experts.56.gate_proj", "model.layers.70.mlp.experts.57.gate_proj", "model.layers.70.mlp.experts.58.gate_proj", "model.layers.70.mlp.experts.59.gate_proj", "model.layers.70.mlp.experts.60.gate_proj", "model.layers.70.mlp.experts.61.gate_proj", "model.layers.70.mlp.experts.62.gate_proj", "model.layers.70.mlp.experts.63.gate_proj", "model.layers.70.mlp.experts.64.gate_proj", "model.layers.70.mlp.experts.65.gate_proj", "model.layers.70.mlp.experts.66.gate_proj", "model.layers.70.mlp.experts.67.gate_proj", "model.layers.70.mlp.experts.68.gate_proj", "model.layers.70.mlp.experts.69.gate_proj", "model.layers.70.mlp.experts.70.gate_proj", "model.layers.70.mlp.experts.71.gate_proj", "model.layers.70.mlp.experts.72.gate_proj", "model.layers.70.mlp.experts.73.gate_proj", "model.layers.70.mlp.experts.74.gate_proj", "model.layers.70.mlp.experts.75.gate_proj", "model.layers.70.mlp.experts.76.gate_proj", "model.layers.70.mlp.experts.77.gate_proj", "model.layers.70.mlp.experts.78.gate_proj", "model.layers.70.mlp.experts.79.gate_proj", "model.layers.70.mlp.experts.80.gate_proj", "model.layers.70.mlp.experts.81.gate_proj", "model.layers.70.mlp.experts.82.gate_proj", "model.layers.70.mlp.experts.83.gate_proj", "model.layers.70.mlp.experts.84.gate_proj", "model.layers.70.mlp.experts.85.gate_proj", "model.layers.70.mlp.experts.86.gate_proj", "model.layers.70.mlp.experts.87.gate_proj", "model.layers.70.mlp.experts.88.gate_proj", "model.layers.70.mlp.experts.89.gate_proj", "model.layers.70.mlp.experts.90.gate_proj", "model.layers.70.mlp.experts.91.gate_proj", "model.layers.70.mlp.experts.92.gate_proj", "model.layers.70.mlp.experts.93.gate_proj", "model.layers.70.mlp.experts.94.gate_proj", "model.layers.70.mlp.experts.95.gate_proj", "model.layers.70.mlp.experts.96.gate_proj", "model.layers.70.mlp.experts.97.gate_proj", "model.layers.70.mlp.experts.98.gate_proj", "model.layers.70.mlp.experts.99.gate_proj", "model.layers.70.mlp.experts.100.gate_proj", "model.layers.70.mlp.experts.101.gate_proj", "model.layers.70.mlp.experts.102.gate_proj", "model.layers.70.mlp.experts.103.gate_proj", "model.layers.70.mlp.experts.104.gate_proj", "model.layers.70.mlp.experts.105.gate_proj", "model.layers.70.mlp.experts.106.gate_proj", "model.layers.70.mlp.experts.107.gate_proj", "model.layers.70.mlp.experts.108.gate_proj", "model.layers.70.mlp.experts.109.gate_proj", "model.layers.70.mlp.experts.110.gate_proj", "model.layers.70.mlp.experts.111.gate_proj", "model.layers.70.mlp.experts.112.gate_proj", "model.layers.70.mlp.experts.113.gate_proj", "model.layers.70.mlp.experts.114.gate_proj", "model.layers.70.mlp.experts.115.gate_proj", "model.layers.70.mlp.experts.116.gate_proj", "model.layers.70.mlp.experts.117.gate_proj", "model.layers.70.mlp.experts.118.gate_proj", "model.layers.70.mlp.experts.119.gate_proj", "model.layers.70.mlp.experts.120.gate_proj", "model.layers.70.mlp.experts.121.gate_proj", "model.layers.70.mlp.experts.122.gate_proj", "model.layers.70.mlp.experts.123.gate_proj", "model.layers.70.mlp.experts.124.gate_proj", "model.layers.70.mlp.experts.125.gate_proj", "model.layers.70.mlp.experts.126.gate_proj", "model.layers.70.mlp.experts.127.gate_proj", "model.layers.70.mlp.experts.128.gate_proj", "model.layers.70.mlp.experts.129.gate_proj", "model.layers.70.mlp.experts.130.gate_proj", "model.layers.70.mlp.experts.131.gate_proj", "model.layers.70.mlp.experts.132.gate_proj", "model.layers.70.mlp.experts.133.gate_proj", "model.layers.70.mlp.experts.134.gate_proj", "model.layers.70.mlp.experts.135.gate_proj", "model.layers.70.mlp.experts.136.gate_proj", "model.layers.70.mlp.experts.137.gate_proj", "model.layers.70.mlp.experts.138.gate_proj", "model.layers.70.mlp.experts.139.gate_proj", "model.layers.70.mlp.experts.140.gate_proj", "model.layers.70.mlp.experts.141.gate_proj", "model.layers.70.mlp.experts.142.gate_proj", "model.layers.70.mlp.experts.143.gate_proj", "model.layers.70.mlp.experts.144.gate_proj", "model.layers.70.mlp.experts.145.gate_proj", "model.layers.70.mlp.experts.146.gate_proj", "model.layers.70.mlp.experts.147.gate_proj", "model.layers.70.mlp.experts.148.gate_proj", "model.layers.70.mlp.experts.149.gate_proj", "model.layers.70.mlp.experts.150.gate_proj", "model.layers.70.mlp.experts.151.gate_proj", "model.layers.70.mlp.experts.152.gate_proj", "model.layers.70.mlp.experts.153.gate_proj", "model.layers.70.mlp.experts.154.gate_proj", "model.layers.70.mlp.experts.155.gate_proj", "model.layers.70.mlp.experts.156.gate_proj", "model.layers.70.mlp.experts.157.gate_proj", "model.layers.70.mlp.experts.158.gate_proj", "model.layers.70.mlp.experts.159.gate_proj", "model.layers.70.mlp.experts.0.up_proj", "model.layers.70.mlp.experts.1.up_proj", "model.layers.70.mlp.experts.2.up_proj", "model.layers.70.mlp.experts.3.up_proj", "model.layers.70.mlp.experts.4.up_proj", "model.layers.70.mlp.experts.5.up_proj", "model.layers.70.mlp.experts.6.up_proj", "model.layers.70.mlp.experts.7.up_proj", "model.layers.70.mlp.experts.8.up_proj", "model.layers.70.mlp.experts.9.up_proj", "model.layers.70.mlp.experts.10.up_proj", "model.layers.70.mlp.experts.11.up_proj", "model.layers.70.mlp.experts.12.up_proj", "model.layers.70.mlp.experts.13.up_proj", "model.layers.70.mlp.experts.14.up_proj", "model.layers.70.mlp.experts.15.up_proj", "model.layers.70.mlp.experts.16.up_proj", "model.layers.70.mlp.experts.17.up_proj", "model.layers.70.mlp.experts.18.up_proj", "model.layers.70.mlp.experts.19.up_proj", "model.layers.70.mlp.experts.20.up_proj", "model.layers.70.mlp.experts.21.up_proj", "model.layers.70.mlp.experts.22.up_proj", "model.layers.70.mlp.experts.23.up_proj", "model.layers.70.mlp.experts.24.up_proj", "model.layers.70.mlp.experts.25.up_proj", "model.layers.70.mlp.experts.26.up_proj", "model.layers.70.mlp.experts.27.up_proj", "model.layers.70.mlp.experts.28.up_proj", "model.layers.70.mlp.experts.29.up_proj", "model.layers.70.mlp.experts.30.up_proj", "model.layers.70.mlp.experts.31.up_proj", "model.layers.70.mlp.experts.32.up_proj", "model.layers.70.mlp.experts.33.up_proj", "model.layers.70.mlp.experts.34.up_proj", "model.layers.70.mlp.experts.35.up_proj", "model.layers.70.mlp.experts.36.up_proj", "model.layers.70.mlp.experts.37.up_proj", "model.layers.70.mlp.experts.38.up_proj", "model.layers.70.mlp.experts.39.up_proj", "model.layers.70.mlp.experts.40.up_proj", "model.layers.70.mlp.experts.41.up_proj", "model.layers.70.mlp.experts.42.up_proj", "model.layers.70.mlp.experts.43.up_proj", "model.layers.70.mlp.experts.44.up_proj", "model.layers.70.mlp.experts.45.up_proj", "model.layers.70.mlp.experts.46.up_proj", "model.layers.70.mlp.experts.47.up_proj", "model.layers.70.mlp.experts.48.up_proj", "model.layers.70.mlp.experts.49.up_proj", "model.layers.70.mlp.experts.50.up_proj", "model.layers.70.mlp.experts.51.up_proj", "model.layers.70.mlp.experts.52.up_proj", "model.layers.70.mlp.experts.53.up_proj", "model.layers.70.mlp.experts.54.up_proj", "model.layers.70.mlp.experts.55.up_proj", "model.layers.70.mlp.experts.56.up_proj", "model.layers.70.mlp.experts.57.up_proj", "model.layers.70.mlp.experts.58.up_proj", "model.layers.70.mlp.experts.59.up_proj", "model.layers.70.mlp.experts.60.up_proj", "model.layers.70.mlp.experts.61.up_proj", "model.layers.70.mlp.experts.62.up_proj", "model.layers.70.mlp.experts.63.up_proj", "model.layers.70.mlp.experts.64.up_proj", "model.layers.70.mlp.experts.65.up_proj", "model.layers.70.mlp.experts.66.up_proj", "model.layers.70.mlp.experts.67.up_proj", "model.layers.70.mlp.experts.68.up_proj", "model.layers.70.mlp.experts.69.up_proj", "model.layers.70.mlp.experts.70.up_proj", "model.layers.70.mlp.experts.71.up_proj", "model.layers.70.mlp.experts.72.up_proj", "model.layers.70.mlp.experts.73.up_proj", "model.layers.70.mlp.experts.74.up_proj", "model.layers.70.mlp.experts.75.up_proj", "model.layers.70.mlp.experts.76.up_proj", "model.layers.70.mlp.experts.77.up_proj", "model.layers.70.mlp.experts.78.up_proj", "model.layers.70.mlp.experts.79.up_proj", "model.layers.70.mlp.experts.80.up_proj", "model.layers.70.mlp.experts.81.up_proj", "model.layers.70.mlp.experts.82.up_proj", "model.layers.70.mlp.experts.83.up_proj", "model.layers.70.mlp.experts.84.up_proj", "model.layers.70.mlp.experts.85.up_proj", "model.layers.70.mlp.experts.86.up_proj", "model.layers.70.mlp.experts.87.up_proj", "model.layers.70.mlp.experts.88.up_proj", "model.layers.70.mlp.experts.89.up_proj", "model.layers.70.mlp.experts.90.up_proj", "model.layers.70.mlp.experts.91.up_proj", "model.layers.70.mlp.experts.92.up_proj", "model.layers.70.mlp.experts.93.up_proj", "model.layers.70.mlp.experts.94.up_proj", "model.layers.70.mlp.experts.95.up_proj", "model.layers.70.mlp.experts.96.up_proj", "model.layers.70.mlp.experts.97.up_proj", "model.layers.70.mlp.experts.98.up_proj", "model.layers.70.mlp.experts.99.up_proj", "model.layers.70.mlp.experts.100.up_proj", "model.layers.70.mlp.experts.101.up_proj", "model.layers.70.mlp.experts.102.up_proj", "model.layers.70.mlp.experts.103.up_proj", "model.layers.70.mlp.experts.104.up_proj", "model.layers.70.mlp.experts.105.up_proj", "model.layers.70.mlp.experts.106.up_proj", "model.layers.70.mlp.experts.107.up_proj", "model.layers.70.mlp.experts.108.up_proj", "model.layers.70.mlp.experts.109.up_proj", "model.layers.70.mlp.experts.110.up_proj", "model.layers.70.mlp.experts.111.up_proj", "model.layers.70.mlp.experts.112.up_proj", "model.layers.70.mlp.experts.113.up_proj", "model.layers.70.mlp.experts.114.up_proj", "model.layers.70.mlp.experts.115.up_proj", "model.layers.70.mlp.experts.116.up_proj", "model.layers.70.mlp.experts.117.up_proj", "model.layers.70.mlp.experts.118.up_proj", "model.layers.70.mlp.experts.119.up_proj", "model.layers.70.mlp.experts.120.up_proj", "model.layers.70.mlp.experts.121.up_proj", "model.layers.70.mlp.experts.122.up_proj", "model.layers.70.mlp.experts.123.up_proj", "model.layers.70.mlp.experts.124.up_proj", "model.layers.70.mlp.experts.125.up_proj", "model.layers.70.mlp.experts.126.up_proj", "model.layers.70.mlp.experts.127.up_proj", "model.layers.70.mlp.experts.128.up_proj", "model.layers.70.mlp.experts.129.up_proj", "model.layers.70.mlp.experts.130.up_proj", "model.layers.70.mlp.experts.131.up_proj", "model.layers.70.mlp.experts.132.up_proj", "model.layers.70.mlp.experts.133.up_proj", "model.layers.70.mlp.experts.134.up_proj", "model.layers.70.mlp.experts.135.up_proj", "model.layers.70.mlp.experts.136.up_proj", "model.layers.70.mlp.experts.137.up_proj", "model.layers.70.mlp.experts.138.up_proj", "model.layers.70.mlp.experts.139.up_proj", "model.layers.70.mlp.experts.140.up_proj", "model.layers.70.mlp.experts.141.up_proj", "model.layers.70.mlp.experts.142.up_proj", "model.layers.70.mlp.experts.143.up_proj", "model.layers.70.mlp.experts.144.up_proj", "model.layers.70.mlp.experts.145.up_proj", "model.layers.70.mlp.experts.146.up_proj", "model.layers.70.mlp.experts.147.up_proj", "model.layers.70.mlp.experts.148.up_proj", "model.layers.70.mlp.experts.149.up_proj", "model.layers.70.mlp.experts.150.up_proj", "model.layers.70.mlp.experts.151.up_proj", "model.layers.70.mlp.experts.152.up_proj", "model.layers.70.mlp.experts.153.up_proj", "model.layers.70.mlp.experts.154.up_proj", "model.layers.70.mlp.experts.155.up_proj", "model.layers.70.mlp.experts.156.up_proj", "model.layers.70.mlp.experts.157.up_proj", "model.layers.70.mlp.experts.158.up_proj", "model.layers.70.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.896874841302677e-06, "dbits": 5033164800 } ] }, { "idx": 422, "layers": [ "model.layers.70.mlp.experts.0.down_proj", "model.layers.70.mlp.experts.1.down_proj", "model.layers.70.mlp.experts.2.down_proj", "model.layers.70.mlp.experts.3.down_proj", "model.layers.70.mlp.experts.4.down_proj", "model.layers.70.mlp.experts.5.down_proj", "model.layers.70.mlp.experts.6.down_proj", "model.layers.70.mlp.experts.7.down_proj", "model.layers.70.mlp.experts.8.down_proj", "model.layers.70.mlp.experts.9.down_proj", "model.layers.70.mlp.experts.10.down_proj", "model.layers.70.mlp.experts.11.down_proj", "model.layers.70.mlp.experts.12.down_proj", "model.layers.70.mlp.experts.13.down_proj", "model.layers.70.mlp.experts.14.down_proj", "model.layers.70.mlp.experts.15.down_proj", "model.layers.70.mlp.experts.16.down_proj", "model.layers.70.mlp.experts.17.down_proj", "model.layers.70.mlp.experts.18.down_proj", "model.layers.70.mlp.experts.19.down_proj", "model.layers.70.mlp.experts.20.down_proj", "model.layers.70.mlp.experts.21.down_proj", "model.layers.70.mlp.experts.22.down_proj", "model.layers.70.mlp.experts.23.down_proj", "model.layers.70.mlp.experts.24.down_proj", "model.layers.70.mlp.experts.25.down_proj", "model.layers.70.mlp.experts.26.down_proj", "model.layers.70.mlp.experts.27.down_proj", "model.layers.70.mlp.experts.28.down_proj", "model.layers.70.mlp.experts.29.down_proj", "model.layers.70.mlp.experts.30.down_proj", "model.layers.70.mlp.experts.31.down_proj", "model.layers.70.mlp.experts.32.down_proj", "model.layers.70.mlp.experts.33.down_proj", "model.layers.70.mlp.experts.34.down_proj", "model.layers.70.mlp.experts.35.down_proj", "model.layers.70.mlp.experts.36.down_proj", "model.layers.70.mlp.experts.37.down_proj", "model.layers.70.mlp.experts.38.down_proj", "model.layers.70.mlp.experts.39.down_proj", "model.layers.70.mlp.experts.40.down_proj", "model.layers.70.mlp.experts.41.down_proj", "model.layers.70.mlp.experts.42.down_proj", "model.layers.70.mlp.experts.43.down_proj", "model.layers.70.mlp.experts.44.down_proj", "model.layers.70.mlp.experts.45.down_proj", "model.layers.70.mlp.experts.46.down_proj", "model.layers.70.mlp.experts.47.down_proj", "model.layers.70.mlp.experts.48.down_proj", "model.layers.70.mlp.experts.49.down_proj", "model.layers.70.mlp.experts.50.down_proj", "model.layers.70.mlp.experts.51.down_proj", "model.layers.70.mlp.experts.52.down_proj", "model.layers.70.mlp.experts.53.down_proj", "model.layers.70.mlp.experts.54.down_proj", "model.layers.70.mlp.experts.55.down_proj", "model.layers.70.mlp.experts.56.down_proj", "model.layers.70.mlp.experts.57.down_proj", "model.layers.70.mlp.experts.58.down_proj", "model.layers.70.mlp.experts.59.down_proj", "model.layers.70.mlp.experts.60.down_proj", "model.layers.70.mlp.experts.61.down_proj", "model.layers.70.mlp.experts.62.down_proj", "model.layers.70.mlp.experts.63.down_proj", "model.layers.70.mlp.experts.64.down_proj", "model.layers.70.mlp.experts.65.down_proj", "model.layers.70.mlp.experts.66.down_proj", "model.layers.70.mlp.experts.67.down_proj", "model.layers.70.mlp.experts.68.down_proj", "model.layers.70.mlp.experts.69.down_proj", "model.layers.70.mlp.experts.70.down_proj", "model.layers.70.mlp.experts.71.down_proj", "model.layers.70.mlp.experts.72.down_proj", "model.layers.70.mlp.experts.73.down_proj", "model.layers.70.mlp.experts.74.down_proj", "model.layers.70.mlp.experts.75.down_proj", "model.layers.70.mlp.experts.76.down_proj", "model.layers.70.mlp.experts.77.down_proj", "model.layers.70.mlp.experts.78.down_proj", "model.layers.70.mlp.experts.79.down_proj", "model.layers.70.mlp.experts.80.down_proj", "model.layers.70.mlp.experts.81.down_proj", "model.layers.70.mlp.experts.82.down_proj", "model.layers.70.mlp.experts.83.down_proj", "model.layers.70.mlp.experts.84.down_proj", "model.layers.70.mlp.experts.85.down_proj", "model.layers.70.mlp.experts.86.down_proj", "model.layers.70.mlp.experts.87.down_proj", "model.layers.70.mlp.experts.88.down_proj", "model.layers.70.mlp.experts.89.down_proj", "model.layers.70.mlp.experts.90.down_proj", "model.layers.70.mlp.experts.91.down_proj", "model.layers.70.mlp.experts.92.down_proj", "model.layers.70.mlp.experts.93.down_proj", "model.layers.70.mlp.experts.94.down_proj", "model.layers.70.mlp.experts.95.down_proj", "model.layers.70.mlp.experts.96.down_proj", "model.layers.70.mlp.experts.97.down_proj", "model.layers.70.mlp.experts.98.down_proj", "model.layers.70.mlp.experts.99.down_proj", "model.layers.70.mlp.experts.100.down_proj", "model.layers.70.mlp.experts.101.down_proj", "model.layers.70.mlp.experts.102.down_proj", "model.layers.70.mlp.experts.103.down_proj", "model.layers.70.mlp.experts.104.down_proj", "model.layers.70.mlp.experts.105.down_proj", "model.layers.70.mlp.experts.106.down_proj", "model.layers.70.mlp.experts.107.down_proj", "model.layers.70.mlp.experts.108.down_proj", "model.layers.70.mlp.experts.109.down_proj", "model.layers.70.mlp.experts.110.down_proj", "model.layers.70.mlp.experts.111.down_proj", "model.layers.70.mlp.experts.112.down_proj", "model.layers.70.mlp.experts.113.down_proj", "model.layers.70.mlp.experts.114.down_proj", "model.layers.70.mlp.experts.115.down_proj", "model.layers.70.mlp.experts.116.down_proj", "model.layers.70.mlp.experts.117.down_proj", "model.layers.70.mlp.experts.118.down_proj", "model.layers.70.mlp.experts.119.down_proj", "model.layers.70.mlp.experts.120.down_proj", "model.layers.70.mlp.experts.121.down_proj", "model.layers.70.mlp.experts.122.down_proj", "model.layers.70.mlp.experts.123.down_proj", "model.layers.70.mlp.experts.124.down_proj", "model.layers.70.mlp.experts.125.down_proj", "model.layers.70.mlp.experts.126.down_proj", "model.layers.70.mlp.experts.127.down_proj", "model.layers.70.mlp.experts.128.down_proj", "model.layers.70.mlp.experts.129.down_proj", "model.layers.70.mlp.experts.130.down_proj", "model.layers.70.mlp.experts.131.down_proj", "model.layers.70.mlp.experts.132.down_proj", "model.layers.70.mlp.experts.133.down_proj", "model.layers.70.mlp.experts.134.down_proj", "model.layers.70.mlp.experts.135.down_proj", "model.layers.70.mlp.experts.136.down_proj", "model.layers.70.mlp.experts.137.down_proj", "model.layers.70.mlp.experts.138.down_proj", "model.layers.70.mlp.experts.139.down_proj", "model.layers.70.mlp.experts.140.down_proj", "model.layers.70.mlp.experts.141.down_proj", "model.layers.70.mlp.experts.142.down_proj", "model.layers.70.mlp.experts.143.down_proj", "model.layers.70.mlp.experts.144.down_proj", "model.layers.70.mlp.experts.145.down_proj", "model.layers.70.mlp.experts.146.down_proj", "model.layers.70.mlp.experts.147.down_proj", "model.layers.70.mlp.experts.148.down_proj", "model.layers.70.mlp.experts.149.down_proj", "model.layers.70.mlp.experts.150.down_proj", "model.layers.70.mlp.experts.151.down_proj", "model.layers.70.mlp.experts.152.down_proj", "model.layers.70.mlp.experts.153.down_proj", "model.layers.70.mlp.experts.154.down_proj", "model.layers.70.mlp.experts.155.down_proj", "model.layers.70.mlp.experts.156.down_proj", "model.layers.70.mlp.experts.157.down_proj", "model.layers.70.mlp.experts.158.down_proj", "model.layers.70.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.64432746917005e-06, "dbits": 2516582400 } ] }, { "idx": 423, "layers": [ "model.layers.71.self_attn.q_proj" ], "candidates": [ { "dkld": -2.3213855456560633e-06, "dbits": 125829120 } ] }, { "idx": 424, "layers": [ "model.layers.71.self_attn.k_proj", "model.layers.71.self_attn.v_proj" ], "candidates": [ { "dkld": 1.5917816199362061e-06, "dbits": 20971520 } ] }, { "idx": 425, "layers": [ "model.layers.71.self_attn.o_proj" ], "candidates": [ { "dkld": -3.7666468415413663e-07, "dbits": 125829120 } ] }, { "idx": 426, "layers": [ "model.layers.71.mlp.shared_experts.gate_proj", "model.layers.71.mlp.shared_experts.up_proj", "model.layers.71.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 5.83894434385006e-06, "dbits": 47185920 } ] }, { "idx": 427, "layers": [ "model.layers.71.mlp.experts.0.gate_proj", "model.layers.71.mlp.experts.1.gate_proj", "model.layers.71.mlp.experts.2.gate_proj", "model.layers.71.mlp.experts.3.gate_proj", "model.layers.71.mlp.experts.4.gate_proj", "model.layers.71.mlp.experts.5.gate_proj", "model.layers.71.mlp.experts.6.gate_proj", "model.layers.71.mlp.experts.7.gate_proj", "model.layers.71.mlp.experts.8.gate_proj", "model.layers.71.mlp.experts.9.gate_proj", "model.layers.71.mlp.experts.10.gate_proj", "model.layers.71.mlp.experts.11.gate_proj", "model.layers.71.mlp.experts.12.gate_proj", "model.layers.71.mlp.experts.13.gate_proj", "model.layers.71.mlp.experts.14.gate_proj", "model.layers.71.mlp.experts.15.gate_proj", "model.layers.71.mlp.experts.16.gate_proj", "model.layers.71.mlp.experts.17.gate_proj", "model.layers.71.mlp.experts.18.gate_proj", "model.layers.71.mlp.experts.19.gate_proj", "model.layers.71.mlp.experts.20.gate_proj", "model.layers.71.mlp.experts.21.gate_proj", "model.layers.71.mlp.experts.22.gate_proj", "model.layers.71.mlp.experts.23.gate_proj", "model.layers.71.mlp.experts.24.gate_proj", "model.layers.71.mlp.experts.25.gate_proj", "model.layers.71.mlp.experts.26.gate_proj", "model.layers.71.mlp.experts.27.gate_proj", "model.layers.71.mlp.experts.28.gate_proj", "model.layers.71.mlp.experts.29.gate_proj", "model.layers.71.mlp.experts.30.gate_proj", "model.layers.71.mlp.experts.31.gate_proj", "model.layers.71.mlp.experts.32.gate_proj", "model.layers.71.mlp.experts.33.gate_proj", "model.layers.71.mlp.experts.34.gate_proj", "model.layers.71.mlp.experts.35.gate_proj", "model.layers.71.mlp.experts.36.gate_proj", "model.layers.71.mlp.experts.37.gate_proj", "model.layers.71.mlp.experts.38.gate_proj", "model.layers.71.mlp.experts.39.gate_proj", "model.layers.71.mlp.experts.40.gate_proj", "model.layers.71.mlp.experts.41.gate_proj", "model.layers.71.mlp.experts.42.gate_proj", "model.layers.71.mlp.experts.43.gate_proj", "model.layers.71.mlp.experts.44.gate_proj", "model.layers.71.mlp.experts.45.gate_proj", "model.layers.71.mlp.experts.46.gate_proj", "model.layers.71.mlp.experts.47.gate_proj", "model.layers.71.mlp.experts.48.gate_proj", "model.layers.71.mlp.experts.49.gate_proj", "model.layers.71.mlp.experts.50.gate_proj", "model.layers.71.mlp.experts.51.gate_proj", "model.layers.71.mlp.experts.52.gate_proj", "model.layers.71.mlp.experts.53.gate_proj", "model.layers.71.mlp.experts.54.gate_proj", "model.layers.71.mlp.experts.55.gate_proj", "model.layers.71.mlp.experts.56.gate_proj", "model.layers.71.mlp.experts.57.gate_proj", "model.layers.71.mlp.experts.58.gate_proj", "model.layers.71.mlp.experts.59.gate_proj", "model.layers.71.mlp.experts.60.gate_proj", "model.layers.71.mlp.experts.61.gate_proj", "model.layers.71.mlp.experts.62.gate_proj", "model.layers.71.mlp.experts.63.gate_proj", "model.layers.71.mlp.experts.64.gate_proj", "model.layers.71.mlp.experts.65.gate_proj", "model.layers.71.mlp.experts.66.gate_proj", "model.layers.71.mlp.experts.67.gate_proj", "model.layers.71.mlp.experts.68.gate_proj", "model.layers.71.mlp.experts.69.gate_proj", "model.layers.71.mlp.experts.70.gate_proj", "model.layers.71.mlp.experts.71.gate_proj", "model.layers.71.mlp.experts.72.gate_proj", "model.layers.71.mlp.experts.73.gate_proj", "model.layers.71.mlp.experts.74.gate_proj", "model.layers.71.mlp.experts.75.gate_proj", "model.layers.71.mlp.experts.76.gate_proj", "model.layers.71.mlp.experts.77.gate_proj", "model.layers.71.mlp.experts.78.gate_proj", "model.layers.71.mlp.experts.79.gate_proj", "model.layers.71.mlp.experts.80.gate_proj", "model.layers.71.mlp.experts.81.gate_proj", "model.layers.71.mlp.experts.82.gate_proj", "model.layers.71.mlp.experts.83.gate_proj", "model.layers.71.mlp.experts.84.gate_proj", "model.layers.71.mlp.experts.85.gate_proj", "model.layers.71.mlp.experts.86.gate_proj", "model.layers.71.mlp.experts.87.gate_proj", "model.layers.71.mlp.experts.88.gate_proj", "model.layers.71.mlp.experts.89.gate_proj", "model.layers.71.mlp.experts.90.gate_proj", "model.layers.71.mlp.experts.91.gate_proj", "model.layers.71.mlp.experts.92.gate_proj", "model.layers.71.mlp.experts.93.gate_proj", "model.layers.71.mlp.experts.94.gate_proj", "model.layers.71.mlp.experts.95.gate_proj", "model.layers.71.mlp.experts.96.gate_proj", "model.layers.71.mlp.experts.97.gate_proj", "model.layers.71.mlp.experts.98.gate_proj", "model.layers.71.mlp.experts.99.gate_proj", "model.layers.71.mlp.experts.100.gate_proj", "model.layers.71.mlp.experts.101.gate_proj", "model.layers.71.mlp.experts.102.gate_proj", "model.layers.71.mlp.experts.103.gate_proj", "model.layers.71.mlp.experts.104.gate_proj", "model.layers.71.mlp.experts.105.gate_proj", "model.layers.71.mlp.experts.106.gate_proj", "model.layers.71.mlp.experts.107.gate_proj", "model.layers.71.mlp.experts.108.gate_proj", "model.layers.71.mlp.experts.109.gate_proj", "model.layers.71.mlp.experts.110.gate_proj", "model.layers.71.mlp.experts.111.gate_proj", "model.layers.71.mlp.experts.112.gate_proj", "model.layers.71.mlp.experts.113.gate_proj", "model.layers.71.mlp.experts.114.gate_proj", "model.layers.71.mlp.experts.115.gate_proj", "model.layers.71.mlp.experts.116.gate_proj", "model.layers.71.mlp.experts.117.gate_proj", "model.layers.71.mlp.experts.118.gate_proj", "model.layers.71.mlp.experts.119.gate_proj", "model.layers.71.mlp.experts.120.gate_proj", "model.layers.71.mlp.experts.121.gate_proj", "model.layers.71.mlp.experts.122.gate_proj", "model.layers.71.mlp.experts.123.gate_proj", "model.layers.71.mlp.experts.124.gate_proj", "model.layers.71.mlp.experts.125.gate_proj", "model.layers.71.mlp.experts.126.gate_proj", "model.layers.71.mlp.experts.127.gate_proj", "model.layers.71.mlp.experts.128.gate_proj", "model.layers.71.mlp.experts.129.gate_proj", "model.layers.71.mlp.experts.130.gate_proj", "model.layers.71.mlp.experts.131.gate_proj", "model.layers.71.mlp.experts.132.gate_proj", "model.layers.71.mlp.experts.133.gate_proj", "model.layers.71.mlp.experts.134.gate_proj", "model.layers.71.mlp.experts.135.gate_proj", "model.layers.71.mlp.experts.136.gate_proj", "model.layers.71.mlp.experts.137.gate_proj", "model.layers.71.mlp.experts.138.gate_proj", "model.layers.71.mlp.experts.139.gate_proj", "model.layers.71.mlp.experts.140.gate_proj", "model.layers.71.mlp.experts.141.gate_proj", "model.layers.71.mlp.experts.142.gate_proj", "model.layers.71.mlp.experts.143.gate_proj", "model.layers.71.mlp.experts.144.gate_proj", "model.layers.71.mlp.experts.145.gate_proj", "model.layers.71.mlp.experts.146.gate_proj", "model.layers.71.mlp.experts.147.gate_proj", "model.layers.71.mlp.experts.148.gate_proj", "model.layers.71.mlp.experts.149.gate_proj", "model.layers.71.mlp.experts.150.gate_proj", "model.layers.71.mlp.experts.151.gate_proj", "model.layers.71.mlp.experts.152.gate_proj", "model.layers.71.mlp.experts.153.gate_proj", "model.layers.71.mlp.experts.154.gate_proj", "model.layers.71.mlp.experts.155.gate_proj", "model.layers.71.mlp.experts.156.gate_proj", "model.layers.71.mlp.experts.157.gate_proj", "model.layers.71.mlp.experts.158.gate_proj", "model.layers.71.mlp.experts.159.gate_proj", "model.layers.71.mlp.experts.0.up_proj", "model.layers.71.mlp.experts.1.up_proj", "model.layers.71.mlp.experts.2.up_proj", "model.layers.71.mlp.experts.3.up_proj", "model.layers.71.mlp.experts.4.up_proj", "model.layers.71.mlp.experts.5.up_proj", "model.layers.71.mlp.experts.6.up_proj", "model.layers.71.mlp.experts.7.up_proj", "model.layers.71.mlp.experts.8.up_proj", "model.layers.71.mlp.experts.9.up_proj", "model.layers.71.mlp.experts.10.up_proj", "model.layers.71.mlp.experts.11.up_proj", "model.layers.71.mlp.experts.12.up_proj", "model.layers.71.mlp.experts.13.up_proj", "model.layers.71.mlp.experts.14.up_proj", "model.layers.71.mlp.experts.15.up_proj", "model.layers.71.mlp.experts.16.up_proj", "model.layers.71.mlp.experts.17.up_proj", "model.layers.71.mlp.experts.18.up_proj", "model.layers.71.mlp.experts.19.up_proj", "model.layers.71.mlp.experts.20.up_proj", "model.layers.71.mlp.experts.21.up_proj", "model.layers.71.mlp.experts.22.up_proj", "model.layers.71.mlp.experts.23.up_proj", "model.layers.71.mlp.experts.24.up_proj", "model.layers.71.mlp.experts.25.up_proj", "model.layers.71.mlp.experts.26.up_proj", "model.layers.71.mlp.experts.27.up_proj", "model.layers.71.mlp.experts.28.up_proj", "model.layers.71.mlp.experts.29.up_proj", "model.layers.71.mlp.experts.30.up_proj", "model.layers.71.mlp.experts.31.up_proj", "model.layers.71.mlp.experts.32.up_proj", "model.layers.71.mlp.experts.33.up_proj", "model.layers.71.mlp.experts.34.up_proj", "model.layers.71.mlp.experts.35.up_proj", "model.layers.71.mlp.experts.36.up_proj", "model.layers.71.mlp.experts.37.up_proj", "model.layers.71.mlp.experts.38.up_proj", "model.layers.71.mlp.experts.39.up_proj", "model.layers.71.mlp.experts.40.up_proj", "model.layers.71.mlp.experts.41.up_proj", "model.layers.71.mlp.experts.42.up_proj", "model.layers.71.mlp.experts.43.up_proj", "model.layers.71.mlp.experts.44.up_proj", "model.layers.71.mlp.experts.45.up_proj", "model.layers.71.mlp.experts.46.up_proj", "model.layers.71.mlp.experts.47.up_proj", "model.layers.71.mlp.experts.48.up_proj", "model.layers.71.mlp.experts.49.up_proj", "model.layers.71.mlp.experts.50.up_proj", "model.layers.71.mlp.experts.51.up_proj", "model.layers.71.mlp.experts.52.up_proj", "model.layers.71.mlp.experts.53.up_proj", "model.layers.71.mlp.experts.54.up_proj", "model.layers.71.mlp.experts.55.up_proj", "model.layers.71.mlp.experts.56.up_proj", "model.layers.71.mlp.experts.57.up_proj", "model.layers.71.mlp.experts.58.up_proj", "model.layers.71.mlp.experts.59.up_proj", "model.layers.71.mlp.experts.60.up_proj", "model.layers.71.mlp.experts.61.up_proj", "model.layers.71.mlp.experts.62.up_proj", "model.layers.71.mlp.experts.63.up_proj", "model.layers.71.mlp.experts.64.up_proj", "model.layers.71.mlp.experts.65.up_proj", "model.layers.71.mlp.experts.66.up_proj", "model.layers.71.mlp.experts.67.up_proj", "model.layers.71.mlp.experts.68.up_proj", "model.layers.71.mlp.experts.69.up_proj", "model.layers.71.mlp.experts.70.up_proj", "model.layers.71.mlp.experts.71.up_proj", "model.layers.71.mlp.experts.72.up_proj", "model.layers.71.mlp.experts.73.up_proj", "model.layers.71.mlp.experts.74.up_proj", "model.layers.71.mlp.experts.75.up_proj", "model.layers.71.mlp.experts.76.up_proj", "model.layers.71.mlp.experts.77.up_proj", "model.layers.71.mlp.experts.78.up_proj", "model.layers.71.mlp.experts.79.up_proj", "model.layers.71.mlp.experts.80.up_proj", "model.layers.71.mlp.experts.81.up_proj", "model.layers.71.mlp.experts.82.up_proj", "model.layers.71.mlp.experts.83.up_proj", "model.layers.71.mlp.experts.84.up_proj", "model.layers.71.mlp.experts.85.up_proj", "model.layers.71.mlp.experts.86.up_proj", "model.layers.71.mlp.experts.87.up_proj", "model.layers.71.mlp.experts.88.up_proj", "model.layers.71.mlp.experts.89.up_proj", "model.layers.71.mlp.experts.90.up_proj", "model.layers.71.mlp.experts.91.up_proj", "model.layers.71.mlp.experts.92.up_proj", "model.layers.71.mlp.experts.93.up_proj", "model.layers.71.mlp.experts.94.up_proj", "model.layers.71.mlp.experts.95.up_proj", "model.layers.71.mlp.experts.96.up_proj", "model.layers.71.mlp.experts.97.up_proj", "model.layers.71.mlp.experts.98.up_proj", "model.layers.71.mlp.experts.99.up_proj", "model.layers.71.mlp.experts.100.up_proj", "model.layers.71.mlp.experts.101.up_proj", "model.layers.71.mlp.experts.102.up_proj", "model.layers.71.mlp.experts.103.up_proj", "model.layers.71.mlp.experts.104.up_proj", "model.layers.71.mlp.experts.105.up_proj", "model.layers.71.mlp.experts.106.up_proj", "model.layers.71.mlp.experts.107.up_proj", "model.layers.71.mlp.experts.108.up_proj", "model.layers.71.mlp.experts.109.up_proj", "model.layers.71.mlp.experts.110.up_proj", "model.layers.71.mlp.experts.111.up_proj", "model.layers.71.mlp.experts.112.up_proj", "model.layers.71.mlp.experts.113.up_proj", "model.layers.71.mlp.experts.114.up_proj", "model.layers.71.mlp.experts.115.up_proj", "model.layers.71.mlp.experts.116.up_proj", "model.layers.71.mlp.experts.117.up_proj", "model.layers.71.mlp.experts.118.up_proj", "model.layers.71.mlp.experts.119.up_proj", "model.layers.71.mlp.experts.120.up_proj", "model.layers.71.mlp.experts.121.up_proj", "model.layers.71.mlp.experts.122.up_proj", "model.layers.71.mlp.experts.123.up_proj", "model.layers.71.mlp.experts.124.up_proj", "model.layers.71.mlp.experts.125.up_proj", "model.layers.71.mlp.experts.126.up_proj", "model.layers.71.mlp.experts.127.up_proj", "model.layers.71.mlp.experts.128.up_proj", "model.layers.71.mlp.experts.129.up_proj", "model.layers.71.mlp.experts.130.up_proj", "model.layers.71.mlp.experts.131.up_proj", "model.layers.71.mlp.experts.132.up_proj", "model.layers.71.mlp.experts.133.up_proj", "model.layers.71.mlp.experts.134.up_proj", "model.layers.71.mlp.experts.135.up_proj", "model.layers.71.mlp.experts.136.up_proj", "model.layers.71.mlp.experts.137.up_proj", "model.layers.71.mlp.experts.138.up_proj", "model.layers.71.mlp.experts.139.up_proj", "model.layers.71.mlp.experts.140.up_proj", "model.layers.71.mlp.experts.141.up_proj", "model.layers.71.mlp.experts.142.up_proj", "model.layers.71.mlp.experts.143.up_proj", "model.layers.71.mlp.experts.144.up_proj", "model.layers.71.mlp.experts.145.up_proj", "model.layers.71.mlp.experts.146.up_proj", "model.layers.71.mlp.experts.147.up_proj", "model.layers.71.mlp.experts.148.up_proj", "model.layers.71.mlp.experts.149.up_proj", "model.layers.71.mlp.experts.150.up_proj", "model.layers.71.mlp.experts.151.up_proj", "model.layers.71.mlp.experts.152.up_proj", "model.layers.71.mlp.experts.153.up_proj", "model.layers.71.mlp.experts.154.up_proj", "model.layers.71.mlp.experts.155.up_proj", "model.layers.71.mlp.experts.156.up_proj", "model.layers.71.mlp.experts.157.up_proj", "model.layers.71.mlp.experts.158.up_proj", "model.layers.71.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.276722807437225e-06, "dbits": 5033164800 } ] }, { "idx": 428, "layers": [ "model.layers.71.mlp.experts.0.down_proj", "model.layers.71.mlp.experts.1.down_proj", "model.layers.71.mlp.experts.2.down_proj", "model.layers.71.mlp.experts.3.down_proj", "model.layers.71.mlp.experts.4.down_proj", "model.layers.71.mlp.experts.5.down_proj", "model.layers.71.mlp.experts.6.down_proj", "model.layers.71.mlp.experts.7.down_proj", "model.layers.71.mlp.experts.8.down_proj", "model.layers.71.mlp.experts.9.down_proj", "model.layers.71.mlp.experts.10.down_proj", "model.layers.71.mlp.experts.11.down_proj", "model.layers.71.mlp.experts.12.down_proj", "model.layers.71.mlp.experts.13.down_proj", "model.layers.71.mlp.experts.14.down_proj", "model.layers.71.mlp.experts.15.down_proj", "model.layers.71.mlp.experts.16.down_proj", "model.layers.71.mlp.experts.17.down_proj", "model.layers.71.mlp.experts.18.down_proj", "model.layers.71.mlp.experts.19.down_proj", "model.layers.71.mlp.experts.20.down_proj", "model.layers.71.mlp.experts.21.down_proj", "model.layers.71.mlp.experts.22.down_proj", "model.layers.71.mlp.experts.23.down_proj", "model.layers.71.mlp.experts.24.down_proj", "model.layers.71.mlp.experts.25.down_proj", "model.layers.71.mlp.experts.26.down_proj", "model.layers.71.mlp.experts.27.down_proj", "model.layers.71.mlp.experts.28.down_proj", "model.layers.71.mlp.experts.29.down_proj", "model.layers.71.mlp.experts.30.down_proj", "model.layers.71.mlp.experts.31.down_proj", "model.layers.71.mlp.experts.32.down_proj", "model.layers.71.mlp.experts.33.down_proj", "model.layers.71.mlp.experts.34.down_proj", "model.layers.71.mlp.experts.35.down_proj", "model.layers.71.mlp.experts.36.down_proj", "model.layers.71.mlp.experts.37.down_proj", "model.layers.71.mlp.experts.38.down_proj", "model.layers.71.mlp.experts.39.down_proj", "model.layers.71.mlp.experts.40.down_proj", "model.layers.71.mlp.experts.41.down_proj", "model.layers.71.mlp.experts.42.down_proj", "model.layers.71.mlp.experts.43.down_proj", "model.layers.71.mlp.experts.44.down_proj", "model.layers.71.mlp.experts.45.down_proj", "model.layers.71.mlp.experts.46.down_proj", "model.layers.71.mlp.experts.47.down_proj", "model.layers.71.mlp.experts.48.down_proj", "model.layers.71.mlp.experts.49.down_proj", "model.layers.71.mlp.experts.50.down_proj", "model.layers.71.mlp.experts.51.down_proj", "model.layers.71.mlp.experts.52.down_proj", "model.layers.71.mlp.experts.53.down_proj", "model.layers.71.mlp.experts.54.down_proj", "model.layers.71.mlp.experts.55.down_proj", "model.layers.71.mlp.experts.56.down_proj", "model.layers.71.mlp.experts.57.down_proj", "model.layers.71.mlp.experts.58.down_proj", "model.layers.71.mlp.experts.59.down_proj", "model.layers.71.mlp.experts.60.down_proj", "model.layers.71.mlp.experts.61.down_proj", "model.layers.71.mlp.experts.62.down_proj", "model.layers.71.mlp.experts.63.down_proj", "model.layers.71.mlp.experts.64.down_proj", "model.layers.71.mlp.experts.65.down_proj", "model.layers.71.mlp.experts.66.down_proj", "model.layers.71.mlp.experts.67.down_proj", "model.layers.71.mlp.experts.68.down_proj", "model.layers.71.mlp.experts.69.down_proj", "model.layers.71.mlp.experts.70.down_proj", "model.layers.71.mlp.experts.71.down_proj", "model.layers.71.mlp.experts.72.down_proj", "model.layers.71.mlp.experts.73.down_proj", "model.layers.71.mlp.experts.74.down_proj", "model.layers.71.mlp.experts.75.down_proj", "model.layers.71.mlp.experts.76.down_proj", "model.layers.71.mlp.experts.77.down_proj", "model.layers.71.mlp.experts.78.down_proj", "model.layers.71.mlp.experts.79.down_proj", "model.layers.71.mlp.experts.80.down_proj", "model.layers.71.mlp.experts.81.down_proj", "model.layers.71.mlp.experts.82.down_proj", "model.layers.71.mlp.experts.83.down_proj", "model.layers.71.mlp.experts.84.down_proj", "model.layers.71.mlp.experts.85.down_proj", "model.layers.71.mlp.experts.86.down_proj", "model.layers.71.mlp.experts.87.down_proj", "model.layers.71.mlp.experts.88.down_proj", "model.layers.71.mlp.experts.89.down_proj", "model.layers.71.mlp.experts.90.down_proj", "model.layers.71.mlp.experts.91.down_proj", "model.layers.71.mlp.experts.92.down_proj", "model.layers.71.mlp.experts.93.down_proj", "model.layers.71.mlp.experts.94.down_proj", "model.layers.71.mlp.experts.95.down_proj", "model.layers.71.mlp.experts.96.down_proj", "model.layers.71.mlp.experts.97.down_proj", "model.layers.71.mlp.experts.98.down_proj", "model.layers.71.mlp.experts.99.down_proj", "model.layers.71.mlp.experts.100.down_proj", "model.layers.71.mlp.experts.101.down_proj", "model.layers.71.mlp.experts.102.down_proj", "model.layers.71.mlp.experts.103.down_proj", "model.layers.71.mlp.experts.104.down_proj", "model.layers.71.mlp.experts.105.down_proj", "model.layers.71.mlp.experts.106.down_proj", "model.layers.71.mlp.experts.107.down_proj", "model.layers.71.mlp.experts.108.down_proj", "model.layers.71.mlp.experts.109.down_proj", "model.layers.71.mlp.experts.110.down_proj", "model.layers.71.mlp.experts.111.down_proj", "model.layers.71.mlp.experts.112.down_proj", "model.layers.71.mlp.experts.113.down_proj", "model.layers.71.mlp.experts.114.down_proj", "model.layers.71.mlp.experts.115.down_proj", "model.layers.71.mlp.experts.116.down_proj", "model.layers.71.mlp.experts.117.down_proj", "model.layers.71.mlp.experts.118.down_proj", "model.layers.71.mlp.experts.119.down_proj", "model.layers.71.mlp.experts.120.down_proj", "model.layers.71.mlp.experts.121.down_proj", "model.layers.71.mlp.experts.122.down_proj", "model.layers.71.mlp.experts.123.down_proj", "model.layers.71.mlp.experts.124.down_proj", "model.layers.71.mlp.experts.125.down_proj", "model.layers.71.mlp.experts.126.down_proj", "model.layers.71.mlp.experts.127.down_proj", "model.layers.71.mlp.experts.128.down_proj", "model.layers.71.mlp.experts.129.down_proj", "model.layers.71.mlp.experts.130.down_proj", "model.layers.71.mlp.experts.131.down_proj", "model.layers.71.mlp.experts.132.down_proj", "model.layers.71.mlp.experts.133.down_proj", "model.layers.71.mlp.experts.134.down_proj", "model.layers.71.mlp.experts.135.down_proj", "model.layers.71.mlp.experts.136.down_proj", "model.layers.71.mlp.experts.137.down_proj", "model.layers.71.mlp.experts.138.down_proj", "model.layers.71.mlp.experts.139.down_proj", "model.layers.71.mlp.experts.140.down_proj", "model.layers.71.mlp.experts.141.down_proj", "model.layers.71.mlp.experts.142.down_proj", "model.layers.71.mlp.experts.143.down_proj", "model.layers.71.mlp.experts.144.down_proj", "model.layers.71.mlp.experts.145.down_proj", "model.layers.71.mlp.experts.146.down_proj", "model.layers.71.mlp.experts.147.down_proj", "model.layers.71.mlp.experts.148.down_proj", "model.layers.71.mlp.experts.149.down_proj", "model.layers.71.mlp.experts.150.down_proj", "model.layers.71.mlp.experts.151.down_proj", "model.layers.71.mlp.experts.152.down_proj", "model.layers.71.mlp.experts.153.down_proj", "model.layers.71.mlp.experts.154.down_proj", "model.layers.71.mlp.experts.155.down_proj", "model.layers.71.mlp.experts.156.down_proj", "model.layers.71.mlp.experts.157.down_proj", "model.layers.71.mlp.experts.158.down_proj", "model.layers.71.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.6682682801038027e-06, "dbits": 2516582400 } ] }, { "idx": 429, "layers": [ "model.layers.72.self_attn.q_proj" ], "candidates": [ { "dkld": 1.2560107279568694e-06, "dbits": 125829120 } ] }, { "idx": 430, "layers": [ "model.layers.72.self_attn.k_proj", "model.layers.72.self_attn.v_proj" ], "candidates": [ { "dkld": -7.781345630064607e-07, "dbits": 20971520 } ] }, { "idx": 431, "layers": [ "model.layers.72.self_attn.o_proj" ], "candidates": [ { "dkld": 3.0870578484609723e-06, "dbits": 125829120 } ] }, { "idx": 432, "layers": [ "model.layers.72.mlp.shared_experts.gate_proj", "model.layers.72.mlp.shared_experts.up_proj", "model.layers.72.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.468738916330126e-06, "dbits": 47185920 } ] }, { "idx": 433, "layers": [ "model.layers.72.mlp.experts.0.gate_proj", "model.layers.72.mlp.experts.1.gate_proj", "model.layers.72.mlp.experts.2.gate_proj", "model.layers.72.mlp.experts.3.gate_proj", "model.layers.72.mlp.experts.4.gate_proj", "model.layers.72.mlp.experts.5.gate_proj", "model.layers.72.mlp.experts.6.gate_proj", "model.layers.72.mlp.experts.7.gate_proj", "model.layers.72.mlp.experts.8.gate_proj", "model.layers.72.mlp.experts.9.gate_proj", "model.layers.72.mlp.experts.10.gate_proj", "model.layers.72.mlp.experts.11.gate_proj", "model.layers.72.mlp.experts.12.gate_proj", "model.layers.72.mlp.experts.13.gate_proj", "model.layers.72.mlp.experts.14.gate_proj", "model.layers.72.mlp.experts.15.gate_proj", "model.layers.72.mlp.experts.16.gate_proj", "model.layers.72.mlp.experts.17.gate_proj", "model.layers.72.mlp.experts.18.gate_proj", "model.layers.72.mlp.experts.19.gate_proj", "model.layers.72.mlp.experts.20.gate_proj", "model.layers.72.mlp.experts.21.gate_proj", "model.layers.72.mlp.experts.22.gate_proj", "model.layers.72.mlp.experts.23.gate_proj", "model.layers.72.mlp.experts.24.gate_proj", "model.layers.72.mlp.experts.25.gate_proj", "model.layers.72.mlp.experts.26.gate_proj", "model.layers.72.mlp.experts.27.gate_proj", "model.layers.72.mlp.experts.28.gate_proj", "model.layers.72.mlp.experts.29.gate_proj", "model.layers.72.mlp.experts.30.gate_proj", "model.layers.72.mlp.experts.31.gate_proj", "model.layers.72.mlp.experts.32.gate_proj", "model.layers.72.mlp.experts.33.gate_proj", "model.layers.72.mlp.experts.34.gate_proj", "model.layers.72.mlp.experts.35.gate_proj", "model.layers.72.mlp.experts.36.gate_proj", "model.layers.72.mlp.experts.37.gate_proj", "model.layers.72.mlp.experts.38.gate_proj", "model.layers.72.mlp.experts.39.gate_proj", "model.layers.72.mlp.experts.40.gate_proj", "model.layers.72.mlp.experts.41.gate_proj", "model.layers.72.mlp.experts.42.gate_proj", "model.layers.72.mlp.experts.43.gate_proj", "model.layers.72.mlp.experts.44.gate_proj", "model.layers.72.mlp.experts.45.gate_proj", "model.layers.72.mlp.experts.46.gate_proj", "model.layers.72.mlp.experts.47.gate_proj", "model.layers.72.mlp.experts.48.gate_proj", "model.layers.72.mlp.experts.49.gate_proj", "model.layers.72.mlp.experts.50.gate_proj", "model.layers.72.mlp.experts.51.gate_proj", "model.layers.72.mlp.experts.52.gate_proj", "model.layers.72.mlp.experts.53.gate_proj", "model.layers.72.mlp.experts.54.gate_proj", "model.layers.72.mlp.experts.55.gate_proj", "model.layers.72.mlp.experts.56.gate_proj", "model.layers.72.mlp.experts.57.gate_proj", "model.layers.72.mlp.experts.58.gate_proj", "model.layers.72.mlp.experts.59.gate_proj", "model.layers.72.mlp.experts.60.gate_proj", "model.layers.72.mlp.experts.61.gate_proj", "model.layers.72.mlp.experts.62.gate_proj", "model.layers.72.mlp.experts.63.gate_proj", "model.layers.72.mlp.experts.64.gate_proj", "model.layers.72.mlp.experts.65.gate_proj", "model.layers.72.mlp.experts.66.gate_proj", "model.layers.72.mlp.experts.67.gate_proj", "model.layers.72.mlp.experts.68.gate_proj", "model.layers.72.mlp.experts.69.gate_proj", "model.layers.72.mlp.experts.70.gate_proj", "model.layers.72.mlp.experts.71.gate_proj", "model.layers.72.mlp.experts.72.gate_proj", "model.layers.72.mlp.experts.73.gate_proj", "model.layers.72.mlp.experts.74.gate_proj", "model.layers.72.mlp.experts.75.gate_proj", "model.layers.72.mlp.experts.76.gate_proj", "model.layers.72.mlp.experts.77.gate_proj", "model.layers.72.mlp.experts.78.gate_proj", "model.layers.72.mlp.experts.79.gate_proj", "model.layers.72.mlp.experts.80.gate_proj", "model.layers.72.mlp.experts.81.gate_proj", "model.layers.72.mlp.experts.82.gate_proj", "model.layers.72.mlp.experts.83.gate_proj", "model.layers.72.mlp.experts.84.gate_proj", "model.layers.72.mlp.experts.85.gate_proj", "model.layers.72.mlp.experts.86.gate_proj", "model.layers.72.mlp.experts.87.gate_proj", "model.layers.72.mlp.experts.88.gate_proj", "model.layers.72.mlp.experts.89.gate_proj", "model.layers.72.mlp.experts.90.gate_proj", "model.layers.72.mlp.experts.91.gate_proj", "model.layers.72.mlp.experts.92.gate_proj", "model.layers.72.mlp.experts.93.gate_proj", "model.layers.72.mlp.experts.94.gate_proj", "model.layers.72.mlp.experts.95.gate_proj", "model.layers.72.mlp.experts.96.gate_proj", "model.layers.72.mlp.experts.97.gate_proj", "model.layers.72.mlp.experts.98.gate_proj", "model.layers.72.mlp.experts.99.gate_proj", "model.layers.72.mlp.experts.100.gate_proj", "model.layers.72.mlp.experts.101.gate_proj", "model.layers.72.mlp.experts.102.gate_proj", "model.layers.72.mlp.experts.103.gate_proj", "model.layers.72.mlp.experts.104.gate_proj", "model.layers.72.mlp.experts.105.gate_proj", "model.layers.72.mlp.experts.106.gate_proj", "model.layers.72.mlp.experts.107.gate_proj", "model.layers.72.mlp.experts.108.gate_proj", "model.layers.72.mlp.experts.109.gate_proj", "model.layers.72.mlp.experts.110.gate_proj", "model.layers.72.mlp.experts.111.gate_proj", "model.layers.72.mlp.experts.112.gate_proj", "model.layers.72.mlp.experts.113.gate_proj", "model.layers.72.mlp.experts.114.gate_proj", "model.layers.72.mlp.experts.115.gate_proj", "model.layers.72.mlp.experts.116.gate_proj", "model.layers.72.mlp.experts.117.gate_proj", "model.layers.72.mlp.experts.118.gate_proj", "model.layers.72.mlp.experts.119.gate_proj", "model.layers.72.mlp.experts.120.gate_proj", "model.layers.72.mlp.experts.121.gate_proj", "model.layers.72.mlp.experts.122.gate_proj", "model.layers.72.mlp.experts.123.gate_proj", "model.layers.72.mlp.experts.124.gate_proj", "model.layers.72.mlp.experts.125.gate_proj", "model.layers.72.mlp.experts.126.gate_proj", "model.layers.72.mlp.experts.127.gate_proj", "model.layers.72.mlp.experts.128.gate_proj", "model.layers.72.mlp.experts.129.gate_proj", "model.layers.72.mlp.experts.130.gate_proj", "model.layers.72.mlp.experts.131.gate_proj", "model.layers.72.mlp.experts.132.gate_proj", "model.layers.72.mlp.experts.133.gate_proj", "model.layers.72.mlp.experts.134.gate_proj", "model.layers.72.mlp.experts.135.gate_proj", "model.layers.72.mlp.experts.136.gate_proj", "model.layers.72.mlp.experts.137.gate_proj", "model.layers.72.mlp.experts.138.gate_proj", "model.layers.72.mlp.experts.139.gate_proj", "model.layers.72.mlp.experts.140.gate_proj", "model.layers.72.mlp.experts.141.gate_proj", "model.layers.72.mlp.experts.142.gate_proj", "model.layers.72.mlp.experts.143.gate_proj", "model.layers.72.mlp.experts.144.gate_proj", "model.layers.72.mlp.experts.145.gate_proj", "model.layers.72.mlp.experts.146.gate_proj", "model.layers.72.mlp.experts.147.gate_proj", "model.layers.72.mlp.experts.148.gate_proj", "model.layers.72.mlp.experts.149.gate_proj", "model.layers.72.mlp.experts.150.gate_proj", "model.layers.72.mlp.experts.151.gate_proj", "model.layers.72.mlp.experts.152.gate_proj", "model.layers.72.mlp.experts.153.gate_proj", "model.layers.72.mlp.experts.154.gate_proj", "model.layers.72.mlp.experts.155.gate_proj", "model.layers.72.mlp.experts.156.gate_proj", "model.layers.72.mlp.experts.157.gate_proj", "model.layers.72.mlp.experts.158.gate_proj", "model.layers.72.mlp.experts.159.gate_proj", "model.layers.72.mlp.experts.0.up_proj", "model.layers.72.mlp.experts.1.up_proj", "model.layers.72.mlp.experts.2.up_proj", "model.layers.72.mlp.experts.3.up_proj", "model.layers.72.mlp.experts.4.up_proj", "model.layers.72.mlp.experts.5.up_proj", "model.layers.72.mlp.experts.6.up_proj", "model.layers.72.mlp.experts.7.up_proj", "model.layers.72.mlp.experts.8.up_proj", "model.layers.72.mlp.experts.9.up_proj", "model.layers.72.mlp.experts.10.up_proj", "model.layers.72.mlp.experts.11.up_proj", "model.layers.72.mlp.experts.12.up_proj", "model.layers.72.mlp.experts.13.up_proj", "model.layers.72.mlp.experts.14.up_proj", "model.layers.72.mlp.experts.15.up_proj", "model.layers.72.mlp.experts.16.up_proj", "model.layers.72.mlp.experts.17.up_proj", "model.layers.72.mlp.experts.18.up_proj", "model.layers.72.mlp.experts.19.up_proj", "model.layers.72.mlp.experts.20.up_proj", "model.layers.72.mlp.experts.21.up_proj", "model.layers.72.mlp.experts.22.up_proj", "model.layers.72.mlp.experts.23.up_proj", "model.layers.72.mlp.experts.24.up_proj", "model.layers.72.mlp.experts.25.up_proj", "model.layers.72.mlp.experts.26.up_proj", "model.layers.72.mlp.experts.27.up_proj", "model.layers.72.mlp.experts.28.up_proj", "model.layers.72.mlp.experts.29.up_proj", "model.layers.72.mlp.experts.30.up_proj", "model.layers.72.mlp.experts.31.up_proj", "model.layers.72.mlp.experts.32.up_proj", "model.layers.72.mlp.experts.33.up_proj", "model.layers.72.mlp.experts.34.up_proj", "model.layers.72.mlp.experts.35.up_proj", "model.layers.72.mlp.experts.36.up_proj", "model.layers.72.mlp.experts.37.up_proj", "model.layers.72.mlp.experts.38.up_proj", "model.layers.72.mlp.experts.39.up_proj", "model.layers.72.mlp.experts.40.up_proj", "model.layers.72.mlp.experts.41.up_proj", "model.layers.72.mlp.experts.42.up_proj", "model.layers.72.mlp.experts.43.up_proj", "model.layers.72.mlp.experts.44.up_proj", "model.layers.72.mlp.experts.45.up_proj", "model.layers.72.mlp.experts.46.up_proj", "model.layers.72.mlp.experts.47.up_proj", "model.layers.72.mlp.experts.48.up_proj", "model.layers.72.mlp.experts.49.up_proj", "model.layers.72.mlp.experts.50.up_proj", "model.layers.72.mlp.experts.51.up_proj", "model.layers.72.mlp.experts.52.up_proj", "model.layers.72.mlp.experts.53.up_proj", "model.layers.72.mlp.experts.54.up_proj", "model.layers.72.mlp.experts.55.up_proj", "model.layers.72.mlp.experts.56.up_proj", "model.layers.72.mlp.experts.57.up_proj", "model.layers.72.mlp.experts.58.up_proj", "model.layers.72.mlp.experts.59.up_proj", "model.layers.72.mlp.experts.60.up_proj", "model.layers.72.mlp.experts.61.up_proj", "model.layers.72.mlp.experts.62.up_proj", "model.layers.72.mlp.experts.63.up_proj", "model.layers.72.mlp.experts.64.up_proj", "model.layers.72.mlp.experts.65.up_proj", "model.layers.72.mlp.experts.66.up_proj", "model.layers.72.mlp.experts.67.up_proj", "model.layers.72.mlp.experts.68.up_proj", "model.layers.72.mlp.experts.69.up_proj", "model.layers.72.mlp.experts.70.up_proj", "model.layers.72.mlp.experts.71.up_proj", "model.layers.72.mlp.experts.72.up_proj", "model.layers.72.mlp.experts.73.up_proj", "model.layers.72.mlp.experts.74.up_proj", "model.layers.72.mlp.experts.75.up_proj", "model.layers.72.mlp.experts.76.up_proj", "model.layers.72.mlp.experts.77.up_proj", "model.layers.72.mlp.experts.78.up_proj", "model.layers.72.mlp.experts.79.up_proj", "model.layers.72.mlp.experts.80.up_proj", "model.layers.72.mlp.experts.81.up_proj", "model.layers.72.mlp.experts.82.up_proj", "model.layers.72.mlp.experts.83.up_proj", "model.layers.72.mlp.experts.84.up_proj", "model.layers.72.mlp.experts.85.up_proj", "model.layers.72.mlp.experts.86.up_proj", "model.layers.72.mlp.experts.87.up_proj", "model.layers.72.mlp.experts.88.up_proj", "model.layers.72.mlp.experts.89.up_proj", "model.layers.72.mlp.experts.90.up_proj", "model.layers.72.mlp.experts.91.up_proj", "model.layers.72.mlp.experts.92.up_proj", "model.layers.72.mlp.experts.93.up_proj", "model.layers.72.mlp.experts.94.up_proj", "model.layers.72.mlp.experts.95.up_proj", "model.layers.72.mlp.experts.96.up_proj", "model.layers.72.mlp.experts.97.up_proj", "model.layers.72.mlp.experts.98.up_proj", "model.layers.72.mlp.experts.99.up_proj", "model.layers.72.mlp.experts.100.up_proj", "model.layers.72.mlp.experts.101.up_proj", "model.layers.72.mlp.experts.102.up_proj", "model.layers.72.mlp.experts.103.up_proj", "model.layers.72.mlp.experts.104.up_proj", "model.layers.72.mlp.experts.105.up_proj", "model.layers.72.mlp.experts.106.up_proj", "model.layers.72.mlp.experts.107.up_proj", "model.layers.72.mlp.experts.108.up_proj", "model.layers.72.mlp.experts.109.up_proj", "model.layers.72.mlp.experts.110.up_proj", "model.layers.72.mlp.experts.111.up_proj", "model.layers.72.mlp.experts.112.up_proj", "model.layers.72.mlp.experts.113.up_proj", "model.layers.72.mlp.experts.114.up_proj", "model.layers.72.mlp.experts.115.up_proj", "model.layers.72.mlp.experts.116.up_proj", "model.layers.72.mlp.experts.117.up_proj", "model.layers.72.mlp.experts.118.up_proj", "model.layers.72.mlp.experts.119.up_proj", "model.layers.72.mlp.experts.120.up_proj", "model.layers.72.mlp.experts.121.up_proj", "model.layers.72.mlp.experts.122.up_proj", "model.layers.72.mlp.experts.123.up_proj", "model.layers.72.mlp.experts.124.up_proj", "model.layers.72.mlp.experts.125.up_proj", "model.layers.72.mlp.experts.126.up_proj", "model.layers.72.mlp.experts.127.up_proj", "model.layers.72.mlp.experts.128.up_proj", "model.layers.72.mlp.experts.129.up_proj", "model.layers.72.mlp.experts.130.up_proj", "model.layers.72.mlp.experts.131.up_proj", "model.layers.72.mlp.experts.132.up_proj", "model.layers.72.mlp.experts.133.up_proj", "model.layers.72.mlp.experts.134.up_proj", "model.layers.72.mlp.experts.135.up_proj", "model.layers.72.mlp.experts.136.up_proj", "model.layers.72.mlp.experts.137.up_proj", "model.layers.72.mlp.experts.138.up_proj", "model.layers.72.mlp.experts.139.up_proj", "model.layers.72.mlp.experts.140.up_proj", "model.layers.72.mlp.experts.141.up_proj", "model.layers.72.mlp.experts.142.up_proj", "model.layers.72.mlp.experts.143.up_proj", "model.layers.72.mlp.experts.144.up_proj", "model.layers.72.mlp.experts.145.up_proj", "model.layers.72.mlp.experts.146.up_proj", "model.layers.72.mlp.experts.147.up_proj", "model.layers.72.mlp.experts.148.up_proj", "model.layers.72.mlp.experts.149.up_proj", "model.layers.72.mlp.experts.150.up_proj", "model.layers.72.mlp.experts.151.up_proj", "model.layers.72.mlp.experts.152.up_proj", "model.layers.72.mlp.experts.153.up_proj", "model.layers.72.mlp.experts.154.up_proj", "model.layers.72.mlp.experts.155.up_proj", "model.layers.72.mlp.experts.156.up_proj", "model.layers.72.mlp.experts.157.up_proj", "model.layers.72.mlp.experts.158.up_proj", "model.layers.72.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.7002388378605247e-06, "dbits": 5033164800 } ] }, { "idx": 434, "layers": [ "model.layers.72.mlp.experts.0.down_proj", "model.layers.72.mlp.experts.1.down_proj", "model.layers.72.mlp.experts.2.down_proj", "model.layers.72.mlp.experts.3.down_proj", "model.layers.72.mlp.experts.4.down_proj", "model.layers.72.mlp.experts.5.down_proj", "model.layers.72.mlp.experts.6.down_proj", "model.layers.72.mlp.experts.7.down_proj", "model.layers.72.mlp.experts.8.down_proj", "model.layers.72.mlp.experts.9.down_proj", "model.layers.72.mlp.experts.10.down_proj", "model.layers.72.mlp.experts.11.down_proj", "model.layers.72.mlp.experts.12.down_proj", "model.layers.72.mlp.experts.13.down_proj", "model.layers.72.mlp.experts.14.down_proj", "model.layers.72.mlp.experts.15.down_proj", "model.layers.72.mlp.experts.16.down_proj", "model.layers.72.mlp.experts.17.down_proj", "model.layers.72.mlp.experts.18.down_proj", "model.layers.72.mlp.experts.19.down_proj", "model.layers.72.mlp.experts.20.down_proj", "model.layers.72.mlp.experts.21.down_proj", "model.layers.72.mlp.experts.22.down_proj", "model.layers.72.mlp.experts.23.down_proj", "model.layers.72.mlp.experts.24.down_proj", "model.layers.72.mlp.experts.25.down_proj", "model.layers.72.mlp.experts.26.down_proj", "model.layers.72.mlp.experts.27.down_proj", "model.layers.72.mlp.experts.28.down_proj", "model.layers.72.mlp.experts.29.down_proj", "model.layers.72.mlp.experts.30.down_proj", "model.layers.72.mlp.experts.31.down_proj", "model.layers.72.mlp.experts.32.down_proj", "model.layers.72.mlp.experts.33.down_proj", "model.layers.72.mlp.experts.34.down_proj", "model.layers.72.mlp.experts.35.down_proj", "model.layers.72.mlp.experts.36.down_proj", "model.layers.72.mlp.experts.37.down_proj", "model.layers.72.mlp.experts.38.down_proj", "model.layers.72.mlp.experts.39.down_proj", "model.layers.72.mlp.experts.40.down_proj", "model.layers.72.mlp.experts.41.down_proj", "model.layers.72.mlp.experts.42.down_proj", "model.layers.72.mlp.experts.43.down_proj", "model.layers.72.mlp.experts.44.down_proj", "model.layers.72.mlp.experts.45.down_proj", "model.layers.72.mlp.experts.46.down_proj", "model.layers.72.mlp.experts.47.down_proj", "model.layers.72.mlp.experts.48.down_proj", "model.layers.72.mlp.experts.49.down_proj", "model.layers.72.mlp.experts.50.down_proj", "model.layers.72.mlp.experts.51.down_proj", "model.layers.72.mlp.experts.52.down_proj", "model.layers.72.mlp.experts.53.down_proj", "model.layers.72.mlp.experts.54.down_proj", "model.layers.72.mlp.experts.55.down_proj", "model.layers.72.mlp.experts.56.down_proj", "model.layers.72.mlp.experts.57.down_proj", "model.layers.72.mlp.experts.58.down_proj", "model.layers.72.mlp.experts.59.down_proj", "model.layers.72.mlp.experts.60.down_proj", "model.layers.72.mlp.experts.61.down_proj", "model.layers.72.mlp.experts.62.down_proj", "model.layers.72.mlp.experts.63.down_proj", "model.layers.72.mlp.experts.64.down_proj", "model.layers.72.mlp.experts.65.down_proj", "model.layers.72.mlp.experts.66.down_proj", "model.layers.72.mlp.experts.67.down_proj", "model.layers.72.mlp.experts.68.down_proj", "model.layers.72.mlp.experts.69.down_proj", "model.layers.72.mlp.experts.70.down_proj", "model.layers.72.mlp.experts.71.down_proj", "model.layers.72.mlp.experts.72.down_proj", "model.layers.72.mlp.experts.73.down_proj", "model.layers.72.mlp.experts.74.down_proj", "model.layers.72.mlp.experts.75.down_proj", "model.layers.72.mlp.experts.76.down_proj", "model.layers.72.mlp.experts.77.down_proj", "model.layers.72.mlp.experts.78.down_proj", "model.layers.72.mlp.experts.79.down_proj", "model.layers.72.mlp.experts.80.down_proj", "model.layers.72.mlp.experts.81.down_proj", "model.layers.72.mlp.experts.82.down_proj", "model.layers.72.mlp.experts.83.down_proj", "model.layers.72.mlp.experts.84.down_proj", "model.layers.72.mlp.experts.85.down_proj", "model.layers.72.mlp.experts.86.down_proj", "model.layers.72.mlp.experts.87.down_proj", "model.layers.72.mlp.experts.88.down_proj", "model.layers.72.mlp.experts.89.down_proj", "model.layers.72.mlp.experts.90.down_proj", "model.layers.72.mlp.experts.91.down_proj", "model.layers.72.mlp.experts.92.down_proj", "model.layers.72.mlp.experts.93.down_proj", "model.layers.72.mlp.experts.94.down_proj", "model.layers.72.mlp.experts.95.down_proj", "model.layers.72.mlp.experts.96.down_proj", "model.layers.72.mlp.experts.97.down_proj", "model.layers.72.mlp.experts.98.down_proj", "model.layers.72.mlp.experts.99.down_proj", "model.layers.72.mlp.experts.100.down_proj", "model.layers.72.mlp.experts.101.down_proj", "model.layers.72.mlp.experts.102.down_proj", "model.layers.72.mlp.experts.103.down_proj", "model.layers.72.mlp.experts.104.down_proj", "model.layers.72.mlp.experts.105.down_proj", "model.layers.72.mlp.experts.106.down_proj", "model.layers.72.mlp.experts.107.down_proj", "model.layers.72.mlp.experts.108.down_proj", "model.layers.72.mlp.experts.109.down_proj", "model.layers.72.mlp.experts.110.down_proj", "model.layers.72.mlp.experts.111.down_proj", "model.layers.72.mlp.experts.112.down_proj", "model.layers.72.mlp.experts.113.down_proj", "model.layers.72.mlp.experts.114.down_proj", "model.layers.72.mlp.experts.115.down_proj", "model.layers.72.mlp.experts.116.down_proj", "model.layers.72.mlp.experts.117.down_proj", "model.layers.72.mlp.experts.118.down_proj", "model.layers.72.mlp.experts.119.down_proj", "model.layers.72.mlp.experts.120.down_proj", "model.layers.72.mlp.experts.121.down_proj", "model.layers.72.mlp.experts.122.down_proj", "model.layers.72.mlp.experts.123.down_proj", "model.layers.72.mlp.experts.124.down_proj", "model.layers.72.mlp.experts.125.down_proj", "model.layers.72.mlp.experts.126.down_proj", "model.layers.72.mlp.experts.127.down_proj", "model.layers.72.mlp.experts.128.down_proj", "model.layers.72.mlp.experts.129.down_proj", "model.layers.72.mlp.experts.130.down_proj", "model.layers.72.mlp.experts.131.down_proj", "model.layers.72.mlp.experts.132.down_proj", "model.layers.72.mlp.experts.133.down_proj", "model.layers.72.mlp.experts.134.down_proj", "model.layers.72.mlp.experts.135.down_proj", "model.layers.72.mlp.experts.136.down_proj", "model.layers.72.mlp.experts.137.down_proj", "model.layers.72.mlp.experts.138.down_proj", "model.layers.72.mlp.experts.139.down_proj", "model.layers.72.mlp.experts.140.down_proj", "model.layers.72.mlp.experts.141.down_proj", "model.layers.72.mlp.experts.142.down_proj", "model.layers.72.mlp.experts.143.down_proj", "model.layers.72.mlp.experts.144.down_proj", "model.layers.72.mlp.experts.145.down_proj", "model.layers.72.mlp.experts.146.down_proj", "model.layers.72.mlp.experts.147.down_proj", "model.layers.72.mlp.experts.148.down_proj", "model.layers.72.mlp.experts.149.down_proj", "model.layers.72.mlp.experts.150.down_proj", "model.layers.72.mlp.experts.151.down_proj", "model.layers.72.mlp.experts.152.down_proj", "model.layers.72.mlp.experts.153.down_proj", "model.layers.72.mlp.experts.154.down_proj", "model.layers.72.mlp.experts.155.down_proj", "model.layers.72.mlp.experts.156.down_proj", "model.layers.72.mlp.experts.157.down_proj", "model.layers.72.mlp.experts.158.down_proj", "model.layers.72.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.2920296285301664e-06, "dbits": 2516582400 } ] }, { "idx": 435, "layers": [ "model.layers.73.self_attn.q_proj" ], "candidates": [ { "dkld": 2.3367523681372187e-06, "dbits": 125829120 } ] }, { "idx": 436, "layers": [ "model.layers.73.self_attn.k_proj", "model.layers.73.self_attn.v_proj" ], "candidates": [ { "dkld": 2.0970677724107883e-06, "dbits": 20971520 } ] }, { "idx": 437, "layers": [ "model.layers.73.self_attn.o_proj" ], "candidates": [ { "dkld": 3.4731579944491603e-06, "dbits": 125829120 } ] }, { "idx": 438, "layers": [ "model.layers.73.mlp.shared_experts.gate_proj", "model.layers.73.mlp.shared_experts.up_proj", "model.layers.73.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.084828495047943e-06, "dbits": 47185920 } ] }, { "idx": 439, "layers": [ "model.layers.73.mlp.experts.0.gate_proj", "model.layers.73.mlp.experts.1.gate_proj", "model.layers.73.mlp.experts.2.gate_proj", "model.layers.73.mlp.experts.3.gate_proj", "model.layers.73.mlp.experts.4.gate_proj", "model.layers.73.mlp.experts.5.gate_proj", "model.layers.73.mlp.experts.6.gate_proj", "model.layers.73.mlp.experts.7.gate_proj", "model.layers.73.mlp.experts.8.gate_proj", "model.layers.73.mlp.experts.9.gate_proj", "model.layers.73.mlp.experts.10.gate_proj", "model.layers.73.mlp.experts.11.gate_proj", "model.layers.73.mlp.experts.12.gate_proj", "model.layers.73.mlp.experts.13.gate_proj", "model.layers.73.mlp.experts.14.gate_proj", "model.layers.73.mlp.experts.15.gate_proj", "model.layers.73.mlp.experts.16.gate_proj", "model.layers.73.mlp.experts.17.gate_proj", "model.layers.73.mlp.experts.18.gate_proj", "model.layers.73.mlp.experts.19.gate_proj", "model.layers.73.mlp.experts.20.gate_proj", "model.layers.73.mlp.experts.21.gate_proj", "model.layers.73.mlp.experts.22.gate_proj", "model.layers.73.mlp.experts.23.gate_proj", "model.layers.73.mlp.experts.24.gate_proj", "model.layers.73.mlp.experts.25.gate_proj", "model.layers.73.mlp.experts.26.gate_proj", "model.layers.73.mlp.experts.27.gate_proj", "model.layers.73.mlp.experts.28.gate_proj", "model.layers.73.mlp.experts.29.gate_proj", "model.layers.73.mlp.experts.30.gate_proj", "model.layers.73.mlp.experts.31.gate_proj", "model.layers.73.mlp.experts.32.gate_proj", "model.layers.73.mlp.experts.33.gate_proj", "model.layers.73.mlp.experts.34.gate_proj", "model.layers.73.mlp.experts.35.gate_proj", "model.layers.73.mlp.experts.36.gate_proj", "model.layers.73.mlp.experts.37.gate_proj", "model.layers.73.mlp.experts.38.gate_proj", "model.layers.73.mlp.experts.39.gate_proj", "model.layers.73.mlp.experts.40.gate_proj", "model.layers.73.mlp.experts.41.gate_proj", "model.layers.73.mlp.experts.42.gate_proj", "model.layers.73.mlp.experts.43.gate_proj", "model.layers.73.mlp.experts.44.gate_proj", "model.layers.73.mlp.experts.45.gate_proj", "model.layers.73.mlp.experts.46.gate_proj", "model.layers.73.mlp.experts.47.gate_proj", "model.layers.73.mlp.experts.48.gate_proj", "model.layers.73.mlp.experts.49.gate_proj", "model.layers.73.mlp.experts.50.gate_proj", "model.layers.73.mlp.experts.51.gate_proj", "model.layers.73.mlp.experts.52.gate_proj", "model.layers.73.mlp.experts.53.gate_proj", "model.layers.73.mlp.experts.54.gate_proj", "model.layers.73.mlp.experts.55.gate_proj", "model.layers.73.mlp.experts.56.gate_proj", "model.layers.73.mlp.experts.57.gate_proj", "model.layers.73.mlp.experts.58.gate_proj", "model.layers.73.mlp.experts.59.gate_proj", "model.layers.73.mlp.experts.60.gate_proj", "model.layers.73.mlp.experts.61.gate_proj", "model.layers.73.mlp.experts.62.gate_proj", "model.layers.73.mlp.experts.63.gate_proj", "model.layers.73.mlp.experts.64.gate_proj", "model.layers.73.mlp.experts.65.gate_proj", "model.layers.73.mlp.experts.66.gate_proj", "model.layers.73.mlp.experts.67.gate_proj", "model.layers.73.mlp.experts.68.gate_proj", "model.layers.73.mlp.experts.69.gate_proj", "model.layers.73.mlp.experts.70.gate_proj", "model.layers.73.mlp.experts.71.gate_proj", "model.layers.73.mlp.experts.72.gate_proj", "model.layers.73.mlp.experts.73.gate_proj", "model.layers.73.mlp.experts.74.gate_proj", "model.layers.73.mlp.experts.75.gate_proj", "model.layers.73.mlp.experts.76.gate_proj", "model.layers.73.mlp.experts.77.gate_proj", "model.layers.73.mlp.experts.78.gate_proj", "model.layers.73.mlp.experts.79.gate_proj", "model.layers.73.mlp.experts.80.gate_proj", "model.layers.73.mlp.experts.81.gate_proj", "model.layers.73.mlp.experts.82.gate_proj", "model.layers.73.mlp.experts.83.gate_proj", "model.layers.73.mlp.experts.84.gate_proj", "model.layers.73.mlp.experts.85.gate_proj", "model.layers.73.mlp.experts.86.gate_proj", "model.layers.73.mlp.experts.87.gate_proj", "model.layers.73.mlp.experts.88.gate_proj", "model.layers.73.mlp.experts.89.gate_proj", "model.layers.73.mlp.experts.90.gate_proj", "model.layers.73.mlp.experts.91.gate_proj", "model.layers.73.mlp.experts.92.gate_proj", "model.layers.73.mlp.experts.93.gate_proj", "model.layers.73.mlp.experts.94.gate_proj", "model.layers.73.mlp.experts.95.gate_proj", "model.layers.73.mlp.experts.96.gate_proj", "model.layers.73.mlp.experts.97.gate_proj", "model.layers.73.mlp.experts.98.gate_proj", "model.layers.73.mlp.experts.99.gate_proj", "model.layers.73.mlp.experts.100.gate_proj", "model.layers.73.mlp.experts.101.gate_proj", "model.layers.73.mlp.experts.102.gate_proj", "model.layers.73.mlp.experts.103.gate_proj", "model.layers.73.mlp.experts.104.gate_proj", "model.layers.73.mlp.experts.105.gate_proj", "model.layers.73.mlp.experts.106.gate_proj", "model.layers.73.mlp.experts.107.gate_proj", "model.layers.73.mlp.experts.108.gate_proj", "model.layers.73.mlp.experts.109.gate_proj", "model.layers.73.mlp.experts.110.gate_proj", "model.layers.73.mlp.experts.111.gate_proj", "model.layers.73.mlp.experts.112.gate_proj", "model.layers.73.mlp.experts.113.gate_proj", "model.layers.73.mlp.experts.114.gate_proj", "model.layers.73.mlp.experts.115.gate_proj", "model.layers.73.mlp.experts.116.gate_proj", "model.layers.73.mlp.experts.117.gate_proj", "model.layers.73.mlp.experts.118.gate_proj", "model.layers.73.mlp.experts.119.gate_proj", "model.layers.73.mlp.experts.120.gate_proj", "model.layers.73.mlp.experts.121.gate_proj", "model.layers.73.mlp.experts.122.gate_proj", "model.layers.73.mlp.experts.123.gate_proj", "model.layers.73.mlp.experts.124.gate_proj", "model.layers.73.mlp.experts.125.gate_proj", "model.layers.73.mlp.experts.126.gate_proj", "model.layers.73.mlp.experts.127.gate_proj", "model.layers.73.mlp.experts.128.gate_proj", "model.layers.73.mlp.experts.129.gate_proj", "model.layers.73.mlp.experts.130.gate_proj", "model.layers.73.mlp.experts.131.gate_proj", "model.layers.73.mlp.experts.132.gate_proj", "model.layers.73.mlp.experts.133.gate_proj", "model.layers.73.mlp.experts.134.gate_proj", "model.layers.73.mlp.experts.135.gate_proj", "model.layers.73.mlp.experts.136.gate_proj", "model.layers.73.mlp.experts.137.gate_proj", "model.layers.73.mlp.experts.138.gate_proj", "model.layers.73.mlp.experts.139.gate_proj", "model.layers.73.mlp.experts.140.gate_proj", "model.layers.73.mlp.experts.141.gate_proj", "model.layers.73.mlp.experts.142.gate_proj", "model.layers.73.mlp.experts.143.gate_proj", "model.layers.73.mlp.experts.144.gate_proj", "model.layers.73.mlp.experts.145.gate_proj", "model.layers.73.mlp.experts.146.gate_proj", "model.layers.73.mlp.experts.147.gate_proj", "model.layers.73.mlp.experts.148.gate_proj", "model.layers.73.mlp.experts.149.gate_proj", "model.layers.73.mlp.experts.150.gate_proj", "model.layers.73.mlp.experts.151.gate_proj", "model.layers.73.mlp.experts.152.gate_proj", "model.layers.73.mlp.experts.153.gate_proj", "model.layers.73.mlp.experts.154.gate_proj", "model.layers.73.mlp.experts.155.gate_proj", "model.layers.73.mlp.experts.156.gate_proj", "model.layers.73.mlp.experts.157.gate_proj", "model.layers.73.mlp.experts.158.gate_proj", "model.layers.73.mlp.experts.159.gate_proj", "model.layers.73.mlp.experts.0.up_proj", "model.layers.73.mlp.experts.1.up_proj", "model.layers.73.mlp.experts.2.up_proj", "model.layers.73.mlp.experts.3.up_proj", "model.layers.73.mlp.experts.4.up_proj", "model.layers.73.mlp.experts.5.up_proj", "model.layers.73.mlp.experts.6.up_proj", "model.layers.73.mlp.experts.7.up_proj", "model.layers.73.mlp.experts.8.up_proj", "model.layers.73.mlp.experts.9.up_proj", "model.layers.73.mlp.experts.10.up_proj", "model.layers.73.mlp.experts.11.up_proj", "model.layers.73.mlp.experts.12.up_proj", "model.layers.73.mlp.experts.13.up_proj", "model.layers.73.mlp.experts.14.up_proj", "model.layers.73.mlp.experts.15.up_proj", "model.layers.73.mlp.experts.16.up_proj", "model.layers.73.mlp.experts.17.up_proj", "model.layers.73.mlp.experts.18.up_proj", "model.layers.73.mlp.experts.19.up_proj", "model.layers.73.mlp.experts.20.up_proj", "model.layers.73.mlp.experts.21.up_proj", "model.layers.73.mlp.experts.22.up_proj", "model.layers.73.mlp.experts.23.up_proj", "model.layers.73.mlp.experts.24.up_proj", "model.layers.73.mlp.experts.25.up_proj", "model.layers.73.mlp.experts.26.up_proj", "model.layers.73.mlp.experts.27.up_proj", "model.layers.73.mlp.experts.28.up_proj", "model.layers.73.mlp.experts.29.up_proj", "model.layers.73.mlp.experts.30.up_proj", "model.layers.73.mlp.experts.31.up_proj", "model.layers.73.mlp.experts.32.up_proj", "model.layers.73.mlp.experts.33.up_proj", "model.layers.73.mlp.experts.34.up_proj", "model.layers.73.mlp.experts.35.up_proj", "model.layers.73.mlp.experts.36.up_proj", "model.layers.73.mlp.experts.37.up_proj", "model.layers.73.mlp.experts.38.up_proj", "model.layers.73.mlp.experts.39.up_proj", "model.layers.73.mlp.experts.40.up_proj", "model.layers.73.mlp.experts.41.up_proj", "model.layers.73.mlp.experts.42.up_proj", "model.layers.73.mlp.experts.43.up_proj", "model.layers.73.mlp.experts.44.up_proj", "model.layers.73.mlp.experts.45.up_proj", "model.layers.73.mlp.experts.46.up_proj", "model.layers.73.mlp.experts.47.up_proj", "model.layers.73.mlp.experts.48.up_proj", "model.layers.73.mlp.experts.49.up_proj", "model.layers.73.mlp.experts.50.up_proj", "model.layers.73.mlp.experts.51.up_proj", "model.layers.73.mlp.experts.52.up_proj", "model.layers.73.mlp.experts.53.up_proj", "model.layers.73.mlp.experts.54.up_proj", "model.layers.73.mlp.experts.55.up_proj", "model.layers.73.mlp.experts.56.up_proj", "model.layers.73.mlp.experts.57.up_proj", "model.layers.73.mlp.experts.58.up_proj", "model.layers.73.mlp.experts.59.up_proj", "model.layers.73.mlp.experts.60.up_proj", "model.layers.73.mlp.experts.61.up_proj", "model.layers.73.mlp.experts.62.up_proj", "model.layers.73.mlp.experts.63.up_proj", "model.layers.73.mlp.experts.64.up_proj", "model.layers.73.mlp.experts.65.up_proj", "model.layers.73.mlp.experts.66.up_proj", "model.layers.73.mlp.experts.67.up_proj", "model.layers.73.mlp.experts.68.up_proj", "model.layers.73.mlp.experts.69.up_proj", "model.layers.73.mlp.experts.70.up_proj", "model.layers.73.mlp.experts.71.up_proj", "model.layers.73.mlp.experts.72.up_proj", "model.layers.73.mlp.experts.73.up_proj", "model.layers.73.mlp.experts.74.up_proj", "model.layers.73.mlp.experts.75.up_proj", "model.layers.73.mlp.experts.76.up_proj", "model.layers.73.mlp.experts.77.up_proj", "model.layers.73.mlp.experts.78.up_proj", "model.layers.73.mlp.experts.79.up_proj", "model.layers.73.mlp.experts.80.up_proj", "model.layers.73.mlp.experts.81.up_proj", "model.layers.73.mlp.experts.82.up_proj", "model.layers.73.mlp.experts.83.up_proj", "model.layers.73.mlp.experts.84.up_proj", "model.layers.73.mlp.experts.85.up_proj", "model.layers.73.mlp.experts.86.up_proj", "model.layers.73.mlp.experts.87.up_proj", "model.layers.73.mlp.experts.88.up_proj", "model.layers.73.mlp.experts.89.up_proj", "model.layers.73.mlp.experts.90.up_proj", "model.layers.73.mlp.experts.91.up_proj", "model.layers.73.mlp.experts.92.up_proj", "model.layers.73.mlp.experts.93.up_proj", "model.layers.73.mlp.experts.94.up_proj", "model.layers.73.mlp.experts.95.up_proj", "model.layers.73.mlp.experts.96.up_proj", "model.layers.73.mlp.experts.97.up_proj", "model.layers.73.mlp.experts.98.up_proj", "model.layers.73.mlp.experts.99.up_proj", "model.layers.73.mlp.experts.100.up_proj", "model.layers.73.mlp.experts.101.up_proj", "model.layers.73.mlp.experts.102.up_proj", "model.layers.73.mlp.experts.103.up_proj", "model.layers.73.mlp.experts.104.up_proj", "model.layers.73.mlp.experts.105.up_proj", "model.layers.73.mlp.experts.106.up_proj", "model.layers.73.mlp.experts.107.up_proj", "model.layers.73.mlp.experts.108.up_proj", "model.layers.73.mlp.experts.109.up_proj", "model.layers.73.mlp.experts.110.up_proj", "model.layers.73.mlp.experts.111.up_proj", "model.layers.73.mlp.experts.112.up_proj", "model.layers.73.mlp.experts.113.up_proj", "model.layers.73.mlp.experts.114.up_proj", "model.layers.73.mlp.experts.115.up_proj", "model.layers.73.mlp.experts.116.up_proj", "model.layers.73.mlp.experts.117.up_proj", "model.layers.73.mlp.experts.118.up_proj", "model.layers.73.mlp.experts.119.up_proj", "model.layers.73.mlp.experts.120.up_proj", "model.layers.73.mlp.experts.121.up_proj", "model.layers.73.mlp.experts.122.up_proj", "model.layers.73.mlp.experts.123.up_proj", "model.layers.73.mlp.experts.124.up_proj", "model.layers.73.mlp.experts.125.up_proj", "model.layers.73.mlp.experts.126.up_proj", "model.layers.73.mlp.experts.127.up_proj", "model.layers.73.mlp.experts.128.up_proj", "model.layers.73.mlp.experts.129.up_proj", "model.layers.73.mlp.experts.130.up_proj", "model.layers.73.mlp.experts.131.up_proj", "model.layers.73.mlp.experts.132.up_proj", "model.layers.73.mlp.experts.133.up_proj", "model.layers.73.mlp.experts.134.up_proj", "model.layers.73.mlp.experts.135.up_proj", "model.layers.73.mlp.experts.136.up_proj", "model.layers.73.mlp.experts.137.up_proj", "model.layers.73.mlp.experts.138.up_proj", "model.layers.73.mlp.experts.139.up_proj", "model.layers.73.mlp.experts.140.up_proj", "model.layers.73.mlp.experts.141.up_proj", "model.layers.73.mlp.experts.142.up_proj", "model.layers.73.mlp.experts.143.up_proj", "model.layers.73.mlp.experts.144.up_proj", "model.layers.73.mlp.experts.145.up_proj", "model.layers.73.mlp.experts.146.up_proj", "model.layers.73.mlp.experts.147.up_proj", "model.layers.73.mlp.experts.148.up_proj", "model.layers.73.mlp.experts.149.up_proj", "model.layers.73.mlp.experts.150.up_proj", "model.layers.73.mlp.experts.151.up_proj", "model.layers.73.mlp.experts.152.up_proj", "model.layers.73.mlp.experts.153.up_proj", "model.layers.73.mlp.experts.154.up_proj", "model.layers.73.mlp.experts.155.up_proj", "model.layers.73.mlp.experts.156.up_proj", "model.layers.73.mlp.experts.157.up_proj", "model.layers.73.mlp.experts.158.up_proj", "model.layers.73.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.756080357357838e-06, "dbits": 5033164800 } ] }, { "idx": 440, "layers": [ "model.layers.73.mlp.experts.0.down_proj", "model.layers.73.mlp.experts.1.down_proj", "model.layers.73.mlp.experts.2.down_proj", "model.layers.73.mlp.experts.3.down_proj", "model.layers.73.mlp.experts.4.down_proj", "model.layers.73.mlp.experts.5.down_proj", "model.layers.73.mlp.experts.6.down_proj", "model.layers.73.mlp.experts.7.down_proj", "model.layers.73.mlp.experts.8.down_proj", "model.layers.73.mlp.experts.9.down_proj", "model.layers.73.mlp.experts.10.down_proj", "model.layers.73.mlp.experts.11.down_proj", "model.layers.73.mlp.experts.12.down_proj", "model.layers.73.mlp.experts.13.down_proj", "model.layers.73.mlp.experts.14.down_proj", "model.layers.73.mlp.experts.15.down_proj", "model.layers.73.mlp.experts.16.down_proj", "model.layers.73.mlp.experts.17.down_proj", "model.layers.73.mlp.experts.18.down_proj", "model.layers.73.mlp.experts.19.down_proj", "model.layers.73.mlp.experts.20.down_proj", "model.layers.73.mlp.experts.21.down_proj", "model.layers.73.mlp.experts.22.down_proj", "model.layers.73.mlp.experts.23.down_proj", "model.layers.73.mlp.experts.24.down_proj", "model.layers.73.mlp.experts.25.down_proj", "model.layers.73.mlp.experts.26.down_proj", "model.layers.73.mlp.experts.27.down_proj", "model.layers.73.mlp.experts.28.down_proj", "model.layers.73.mlp.experts.29.down_proj", "model.layers.73.mlp.experts.30.down_proj", "model.layers.73.mlp.experts.31.down_proj", "model.layers.73.mlp.experts.32.down_proj", "model.layers.73.mlp.experts.33.down_proj", "model.layers.73.mlp.experts.34.down_proj", "model.layers.73.mlp.experts.35.down_proj", "model.layers.73.mlp.experts.36.down_proj", "model.layers.73.mlp.experts.37.down_proj", "model.layers.73.mlp.experts.38.down_proj", "model.layers.73.mlp.experts.39.down_proj", "model.layers.73.mlp.experts.40.down_proj", "model.layers.73.mlp.experts.41.down_proj", "model.layers.73.mlp.experts.42.down_proj", "model.layers.73.mlp.experts.43.down_proj", "model.layers.73.mlp.experts.44.down_proj", "model.layers.73.mlp.experts.45.down_proj", "model.layers.73.mlp.experts.46.down_proj", "model.layers.73.mlp.experts.47.down_proj", "model.layers.73.mlp.experts.48.down_proj", "model.layers.73.mlp.experts.49.down_proj", "model.layers.73.mlp.experts.50.down_proj", "model.layers.73.mlp.experts.51.down_proj", "model.layers.73.mlp.experts.52.down_proj", "model.layers.73.mlp.experts.53.down_proj", "model.layers.73.mlp.experts.54.down_proj", "model.layers.73.mlp.experts.55.down_proj", "model.layers.73.mlp.experts.56.down_proj", "model.layers.73.mlp.experts.57.down_proj", "model.layers.73.mlp.experts.58.down_proj", "model.layers.73.mlp.experts.59.down_proj", "model.layers.73.mlp.experts.60.down_proj", "model.layers.73.mlp.experts.61.down_proj", "model.layers.73.mlp.experts.62.down_proj", "model.layers.73.mlp.experts.63.down_proj", "model.layers.73.mlp.experts.64.down_proj", "model.layers.73.mlp.experts.65.down_proj", "model.layers.73.mlp.experts.66.down_proj", "model.layers.73.mlp.experts.67.down_proj", "model.layers.73.mlp.experts.68.down_proj", "model.layers.73.mlp.experts.69.down_proj", "model.layers.73.mlp.experts.70.down_proj", "model.layers.73.mlp.experts.71.down_proj", "model.layers.73.mlp.experts.72.down_proj", "model.layers.73.mlp.experts.73.down_proj", "model.layers.73.mlp.experts.74.down_proj", "model.layers.73.mlp.experts.75.down_proj", "model.layers.73.mlp.experts.76.down_proj", "model.layers.73.mlp.experts.77.down_proj", "model.layers.73.mlp.experts.78.down_proj", "model.layers.73.mlp.experts.79.down_proj", "model.layers.73.mlp.experts.80.down_proj", "model.layers.73.mlp.experts.81.down_proj", "model.layers.73.mlp.experts.82.down_proj", "model.layers.73.mlp.experts.83.down_proj", "model.layers.73.mlp.experts.84.down_proj", "model.layers.73.mlp.experts.85.down_proj", "model.layers.73.mlp.experts.86.down_proj", "model.layers.73.mlp.experts.87.down_proj", "model.layers.73.mlp.experts.88.down_proj", "model.layers.73.mlp.experts.89.down_proj", "model.layers.73.mlp.experts.90.down_proj", "model.layers.73.mlp.experts.91.down_proj", "model.layers.73.mlp.experts.92.down_proj", "model.layers.73.mlp.experts.93.down_proj", "model.layers.73.mlp.experts.94.down_proj", "model.layers.73.mlp.experts.95.down_proj", "model.layers.73.mlp.experts.96.down_proj", "model.layers.73.mlp.experts.97.down_proj", "model.layers.73.mlp.experts.98.down_proj", "model.layers.73.mlp.experts.99.down_proj", "model.layers.73.mlp.experts.100.down_proj", "model.layers.73.mlp.experts.101.down_proj", "model.layers.73.mlp.experts.102.down_proj", "model.layers.73.mlp.experts.103.down_proj", "model.layers.73.mlp.experts.104.down_proj", "model.layers.73.mlp.experts.105.down_proj", "model.layers.73.mlp.experts.106.down_proj", "model.layers.73.mlp.experts.107.down_proj", "model.layers.73.mlp.experts.108.down_proj", "model.layers.73.mlp.experts.109.down_proj", "model.layers.73.mlp.experts.110.down_proj", "model.layers.73.mlp.experts.111.down_proj", "model.layers.73.mlp.experts.112.down_proj", "model.layers.73.mlp.experts.113.down_proj", "model.layers.73.mlp.experts.114.down_proj", "model.layers.73.mlp.experts.115.down_proj", "model.layers.73.mlp.experts.116.down_proj", "model.layers.73.mlp.experts.117.down_proj", "model.layers.73.mlp.experts.118.down_proj", "model.layers.73.mlp.experts.119.down_proj", "model.layers.73.mlp.experts.120.down_proj", "model.layers.73.mlp.experts.121.down_proj", "model.layers.73.mlp.experts.122.down_proj", "model.layers.73.mlp.experts.123.down_proj", "model.layers.73.mlp.experts.124.down_proj", "model.layers.73.mlp.experts.125.down_proj", "model.layers.73.mlp.experts.126.down_proj", "model.layers.73.mlp.experts.127.down_proj", "model.layers.73.mlp.experts.128.down_proj", "model.layers.73.mlp.experts.129.down_proj", "model.layers.73.mlp.experts.130.down_proj", "model.layers.73.mlp.experts.131.down_proj", "model.layers.73.mlp.experts.132.down_proj", "model.layers.73.mlp.experts.133.down_proj", "model.layers.73.mlp.experts.134.down_proj", "model.layers.73.mlp.experts.135.down_proj", "model.layers.73.mlp.experts.136.down_proj", "model.layers.73.mlp.experts.137.down_proj", "model.layers.73.mlp.experts.138.down_proj", "model.layers.73.mlp.experts.139.down_proj", "model.layers.73.mlp.experts.140.down_proj", "model.layers.73.mlp.experts.141.down_proj", "model.layers.73.mlp.experts.142.down_proj", "model.layers.73.mlp.experts.143.down_proj", "model.layers.73.mlp.experts.144.down_proj", "model.layers.73.mlp.experts.145.down_proj", "model.layers.73.mlp.experts.146.down_proj", "model.layers.73.mlp.experts.147.down_proj", "model.layers.73.mlp.experts.148.down_proj", "model.layers.73.mlp.experts.149.down_proj", "model.layers.73.mlp.experts.150.down_proj", "model.layers.73.mlp.experts.151.down_proj", "model.layers.73.mlp.experts.152.down_proj", "model.layers.73.mlp.experts.153.down_proj", "model.layers.73.mlp.experts.154.down_proj", "model.layers.73.mlp.experts.155.down_proj", "model.layers.73.mlp.experts.156.down_proj", "model.layers.73.mlp.experts.157.down_proj", "model.layers.73.mlp.experts.158.down_proj", "model.layers.73.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.4746841518208385e-06, "dbits": 2516582400 } ] }, { "idx": 441, "layers": [ "model.layers.74.self_attn.q_proj" ], "candidates": [ { "dkld": 1.037769834510982e-06, "dbits": 125829120 } ] }, { "idx": 442, "layers": [ "model.layers.74.self_attn.k_proj", "model.layers.74.self_attn.v_proj" ], "candidates": [ { "dkld": 4.867662210017486e-06, "dbits": 20971520 } ] }, { "idx": 443, "layers": [ "model.layers.74.self_attn.o_proj" ], "candidates": [ { "dkld": 5.45303919352591e-06, "dbits": 125829120 } ] }, { "idx": 444, "layers": [ "model.layers.74.mlp.shared_experts.gate_proj", "model.layers.74.mlp.shared_experts.up_proj", "model.layers.74.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.9903847714886503e-06, "dbits": 47185920 } ] }, { "idx": 445, "layers": [ "model.layers.74.mlp.experts.0.gate_proj", "model.layers.74.mlp.experts.1.gate_proj", "model.layers.74.mlp.experts.2.gate_proj", "model.layers.74.mlp.experts.3.gate_proj", "model.layers.74.mlp.experts.4.gate_proj", "model.layers.74.mlp.experts.5.gate_proj", "model.layers.74.mlp.experts.6.gate_proj", "model.layers.74.mlp.experts.7.gate_proj", "model.layers.74.mlp.experts.8.gate_proj", "model.layers.74.mlp.experts.9.gate_proj", "model.layers.74.mlp.experts.10.gate_proj", "model.layers.74.mlp.experts.11.gate_proj", "model.layers.74.mlp.experts.12.gate_proj", "model.layers.74.mlp.experts.13.gate_proj", "model.layers.74.mlp.experts.14.gate_proj", "model.layers.74.mlp.experts.15.gate_proj", "model.layers.74.mlp.experts.16.gate_proj", "model.layers.74.mlp.experts.17.gate_proj", "model.layers.74.mlp.experts.18.gate_proj", "model.layers.74.mlp.experts.19.gate_proj", "model.layers.74.mlp.experts.20.gate_proj", "model.layers.74.mlp.experts.21.gate_proj", "model.layers.74.mlp.experts.22.gate_proj", "model.layers.74.mlp.experts.23.gate_proj", "model.layers.74.mlp.experts.24.gate_proj", "model.layers.74.mlp.experts.25.gate_proj", "model.layers.74.mlp.experts.26.gate_proj", "model.layers.74.mlp.experts.27.gate_proj", "model.layers.74.mlp.experts.28.gate_proj", "model.layers.74.mlp.experts.29.gate_proj", "model.layers.74.mlp.experts.30.gate_proj", "model.layers.74.mlp.experts.31.gate_proj", "model.layers.74.mlp.experts.32.gate_proj", "model.layers.74.mlp.experts.33.gate_proj", "model.layers.74.mlp.experts.34.gate_proj", "model.layers.74.mlp.experts.35.gate_proj", "model.layers.74.mlp.experts.36.gate_proj", "model.layers.74.mlp.experts.37.gate_proj", "model.layers.74.mlp.experts.38.gate_proj", "model.layers.74.mlp.experts.39.gate_proj", "model.layers.74.mlp.experts.40.gate_proj", "model.layers.74.mlp.experts.41.gate_proj", "model.layers.74.mlp.experts.42.gate_proj", "model.layers.74.mlp.experts.43.gate_proj", "model.layers.74.mlp.experts.44.gate_proj", "model.layers.74.mlp.experts.45.gate_proj", "model.layers.74.mlp.experts.46.gate_proj", "model.layers.74.mlp.experts.47.gate_proj", "model.layers.74.mlp.experts.48.gate_proj", "model.layers.74.mlp.experts.49.gate_proj", "model.layers.74.mlp.experts.50.gate_proj", "model.layers.74.mlp.experts.51.gate_proj", "model.layers.74.mlp.experts.52.gate_proj", "model.layers.74.mlp.experts.53.gate_proj", "model.layers.74.mlp.experts.54.gate_proj", "model.layers.74.mlp.experts.55.gate_proj", "model.layers.74.mlp.experts.56.gate_proj", "model.layers.74.mlp.experts.57.gate_proj", "model.layers.74.mlp.experts.58.gate_proj", "model.layers.74.mlp.experts.59.gate_proj", "model.layers.74.mlp.experts.60.gate_proj", "model.layers.74.mlp.experts.61.gate_proj", "model.layers.74.mlp.experts.62.gate_proj", "model.layers.74.mlp.experts.63.gate_proj", "model.layers.74.mlp.experts.64.gate_proj", "model.layers.74.mlp.experts.65.gate_proj", "model.layers.74.mlp.experts.66.gate_proj", "model.layers.74.mlp.experts.67.gate_proj", "model.layers.74.mlp.experts.68.gate_proj", "model.layers.74.mlp.experts.69.gate_proj", "model.layers.74.mlp.experts.70.gate_proj", "model.layers.74.mlp.experts.71.gate_proj", "model.layers.74.mlp.experts.72.gate_proj", "model.layers.74.mlp.experts.73.gate_proj", "model.layers.74.mlp.experts.74.gate_proj", "model.layers.74.mlp.experts.75.gate_proj", "model.layers.74.mlp.experts.76.gate_proj", "model.layers.74.mlp.experts.77.gate_proj", "model.layers.74.mlp.experts.78.gate_proj", "model.layers.74.mlp.experts.79.gate_proj", "model.layers.74.mlp.experts.80.gate_proj", "model.layers.74.mlp.experts.81.gate_proj", "model.layers.74.mlp.experts.82.gate_proj", "model.layers.74.mlp.experts.83.gate_proj", "model.layers.74.mlp.experts.84.gate_proj", "model.layers.74.mlp.experts.85.gate_proj", "model.layers.74.mlp.experts.86.gate_proj", "model.layers.74.mlp.experts.87.gate_proj", "model.layers.74.mlp.experts.88.gate_proj", "model.layers.74.mlp.experts.89.gate_proj", "model.layers.74.mlp.experts.90.gate_proj", "model.layers.74.mlp.experts.91.gate_proj", "model.layers.74.mlp.experts.92.gate_proj", "model.layers.74.mlp.experts.93.gate_proj", "model.layers.74.mlp.experts.94.gate_proj", "model.layers.74.mlp.experts.95.gate_proj", "model.layers.74.mlp.experts.96.gate_proj", "model.layers.74.mlp.experts.97.gate_proj", "model.layers.74.mlp.experts.98.gate_proj", "model.layers.74.mlp.experts.99.gate_proj", "model.layers.74.mlp.experts.100.gate_proj", "model.layers.74.mlp.experts.101.gate_proj", "model.layers.74.mlp.experts.102.gate_proj", "model.layers.74.mlp.experts.103.gate_proj", "model.layers.74.mlp.experts.104.gate_proj", "model.layers.74.mlp.experts.105.gate_proj", "model.layers.74.mlp.experts.106.gate_proj", "model.layers.74.mlp.experts.107.gate_proj", "model.layers.74.mlp.experts.108.gate_proj", "model.layers.74.mlp.experts.109.gate_proj", "model.layers.74.mlp.experts.110.gate_proj", "model.layers.74.mlp.experts.111.gate_proj", "model.layers.74.mlp.experts.112.gate_proj", "model.layers.74.mlp.experts.113.gate_proj", "model.layers.74.mlp.experts.114.gate_proj", "model.layers.74.mlp.experts.115.gate_proj", "model.layers.74.mlp.experts.116.gate_proj", "model.layers.74.mlp.experts.117.gate_proj", "model.layers.74.mlp.experts.118.gate_proj", "model.layers.74.mlp.experts.119.gate_proj", "model.layers.74.mlp.experts.120.gate_proj", "model.layers.74.mlp.experts.121.gate_proj", "model.layers.74.mlp.experts.122.gate_proj", "model.layers.74.mlp.experts.123.gate_proj", "model.layers.74.mlp.experts.124.gate_proj", "model.layers.74.mlp.experts.125.gate_proj", "model.layers.74.mlp.experts.126.gate_proj", "model.layers.74.mlp.experts.127.gate_proj", "model.layers.74.mlp.experts.128.gate_proj", "model.layers.74.mlp.experts.129.gate_proj", "model.layers.74.mlp.experts.130.gate_proj", "model.layers.74.mlp.experts.131.gate_proj", "model.layers.74.mlp.experts.132.gate_proj", "model.layers.74.mlp.experts.133.gate_proj", "model.layers.74.mlp.experts.134.gate_proj", "model.layers.74.mlp.experts.135.gate_proj", "model.layers.74.mlp.experts.136.gate_proj", "model.layers.74.mlp.experts.137.gate_proj", "model.layers.74.mlp.experts.138.gate_proj", "model.layers.74.mlp.experts.139.gate_proj", "model.layers.74.mlp.experts.140.gate_proj", "model.layers.74.mlp.experts.141.gate_proj", "model.layers.74.mlp.experts.142.gate_proj", "model.layers.74.mlp.experts.143.gate_proj", "model.layers.74.mlp.experts.144.gate_proj", "model.layers.74.mlp.experts.145.gate_proj", "model.layers.74.mlp.experts.146.gate_proj", "model.layers.74.mlp.experts.147.gate_proj", "model.layers.74.mlp.experts.148.gate_proj", "model.layers.74.mlp.experts.149.gate_proj", "model.layers.74.mlp.experts.150.gate_proj", "model.layers.74.mlp.experts.151.gate_proj", "model.layers.74.mlp.experts.152.gate_proj", "model.layers.74.mlp.experts.153.gate_proj", "model.layers.74.mlp.experts.154.gate_proj", "model.layers.74.mlp.experts.155.gate_proj", "model.layers.74.mlp.experts.156.gate_proj", "model.layers.74.mlp.experts.157.gate_proj", "model.layers.74.mlp.experts.158.gate_proj", "model.layers.74.mlp.experts.159.gate_proj", "model.layers.74.mlp.experts.0.up_proj", "model.layers.74.mlp.experts.1.up_proj", "model.layers.74.mlp.experts.2.up_proj", "model.layers.74.mlp.experts.3.up_proj", "model.layers.74.mlp.experts.4.up_proj", "model.layers.74.mlp.experts.5.up_proj", "model.layers.74.mlp.experts.6.up_proj", "model.layers.74.mlp.experts.7.up_proj", "model.layers.74.mlp.experts.8.up_proj", "model.layers.74.mlp.experts.9.up_proj", "model.layers.74.mlp.experts.10.up_proj", "model.layers.74.mlp.experts.11.up_proj", "model.layers.74.mlp.experts.12.up_proj", "model.layers.74.mlp.experts.13.up_proj", "model.layers.74.mlp.experts.14.up_proj", "model.layers.74.mlp.experts.15.up_proj", "model.layers.74.mlp.experts.16.up_proj", "model.layers.74.mlp.experts.17.up_proj", "model.layers.74.mlp.experts.18.up_proj", "model.layers.74.mlp.experts.19.up_proj", "model.layers.74.mlp.experts.20.up_proj", "model.layers.74.mlp.experts.21.up_proj", "model.layers.74.mlp.experts.22.up_proj", "model.layers.74.mlp.experts.23.up_proj", "model.layers.74.mlp.experts.24.up_proj", "model.layers.74.mlp.experts.25.up_proj", "model.layers.74.mlp.experts.26.up_proj", "model.layers.74.mlp.experts.27.up_proj", "model.layers.74.mlp.experts.28.up_proj", "model.layers.74.mlp.experts.29.up_proj", "model.layers.74.mlp.experts.30.up_proj", "model.layers.74.mlp.experts.31.up_proj", "model.layers.74.mlp.experts.32.up_proj", "model.layers.74.mlp.experts.33.up_proj", "model.layers.74.mlp.experts.34.up_proj", "model.layers.74.mlp.experts.35.up_proj", "model.layers.74.mlp.experts.36.up_proj", "model.layers.74.mlp.experts.37.up_proj", "model.layers.74.mlp.experts.38.up_proj", "model.layers.74.mlp.experts.39.up_proj", "model.layers.74.mlp.experts.40.up_proj", "model.layers.74.mlp.experts.41.up_proj", "model.layers.74.mlp.experts.42.up_proj", "model.layers.74.mlp.experts.43.up_proj", "model.layers.74.mlp.experts.44.up_proj", "model.layers.74.mlp.experts.45.up_proj", "model.layers.74.mlp.experts.46.up_proj", "model.layers.74.mlp.experts.47.up_proj", "model.layers.74.mlp.experts.48.up_proj", "model.layers.74.mlp.experts.49.up_proj", "model.layers.74.mlp.experts.50.up_proj", "model.layers.74.mlp.experts.51.up_proj", "model.layers.74.mlp.experts.52.up_proj", "model.layers.74.mlp.experts.53.up_proj", "model.layers.74.mlp.experts.54.up_proj", "model.layers.74.mlp.experts.55.up_proj", "model.layers.74.mlp.experts.56.up_proj", "model.layers.74.mlp.experts.57.up_proj", "model.layers.74.mlp.experts.58.up_proj", "model.layers.74.mlp.experts.59.up_proj", "model.layers.74.mlp.experts.60.up_proj", "model.layers.74.mlp.experts.61.up_proj", "model.layers.74.mlp.experts.62.up_proj", "model.layers.74.mlp.experts.63.up_proj", "model.layers.74.mlp.experts.64.up_proj", "model.layers.74.mlp.experts.65.up_proj", "model.layers.74.mlp.experts.66.up_proj", "model.layers.74.mlp.experts.67.up_proj", "model.layers.74.mlp.experts.68.up_proj", "model.layers.74.mlp.experts.69.up_proj", "model.layers.74.mlp.experts.70.up_proj", "model.layers.74.mlp.experts.71.up_proj", "model.layers.74.mlp.experts.72.up_proj", "model.layers.74.mlp.experts.73.up_proj", "model.layers.74.mlp.experts.74.up_proj", "model.layers.74.mlp.experts.75.up_proj", "model.layers.74.mlp.experts.76.up_proj", "model.layers.74.mlp.experts.77.up_proj", "model.layers.74.mlp.experts.78.up_proj", "model.layers.74.mlp.experts.79.up_proj", "model.layers.74.mlp.experts.80.up_proj", "model.layers.74.mlp.experts.81.up_proj", "model.layers.74.mlp.experts.82.up_proj", "model.layers.74.mlp.experts.83.up_proj", "model.layers.74.mlp.experts.84.up_proj", "model.layers.74.mlp.experts.85.up_proj", "model.layers.74.mlp.experts.86.up_proj", "model.layers.74.mlp.experts.87.up_proj", "model.layers.74.mlp.experts.88.up_proj", "model.layers.74.mlp.experts.89.up_proj", "model.layers.74.mlp.experts.90.up_proj", "model.layers.74.mlp.experts.91.up_proj", "model.layers.74.mlp.experts.92.up_proj", "model.layers.74.mlp.experts.93.up_proj", "model.layers.74.mlp.experts.94.up_proj", "model.layers.74.mlp.experts.95.up_proj", "model.layers.74.mlp.experts.96.up_proj", "model.layers.74.mlp.experts.97.up_proj", "model.layers.74.mlp.experts.98.up_proj", "model.layers.74.mlp.experts.99.up_proj", "model.layers.74.mlp.experts.100.up_proj", "model.layers.74.mlp.experts.101.up_proj", "model.layers.74.mlp.experts.102.up_proj", "model.layers.74.mlp.experts.103.up_proj", "model.layers.74.mlp.experts.104.up_proj", "model.layers.74.mlp.experts.105.up_proj", "model.layers.74.mlp.experts.106.up_proj", "model.layers.74.mlp.experts.107.up_proj", "model.layers.74.mlp.experts.108.up_proj", "model.layers.74.mlp.experts.109.up_proj", "model.layers.74.mlp.experts.110.up_proj", "model.layers.74.mlp.experts.111.up_proj", "model.layers.74.mlp.experts.112.up_proj", "model.layers.74.mlp.experts.113.up_proj", "model.layers.74.mlp.experts.114.up_proj", "model.layers.74.mlp.experts.115.up_proj", "model.layers.74.mlp.experts.116.up_proj", "model.layers.74.mlp.experts.117.up_proj", "model.layers.74.mlp.experts.118.up_proj", "model.layers.74.mlp.experts.119.up_proj", "model.layers.74.mlp.experts.120.up_proj", "model.layers.74.mlp.experts.121.up_proj", "model.layers.74.mlp.experts.122.up_proj", "model.layers.74.mlp.experts.123.up_proj", "model.layers.74.mlp.experts.124.up_proj", "model.layers.74.mlp.experts.125.up_proj", "model.layers.74.mlp.experts.126.up_proj", "model.layers.74.mlp.experts.127.up_proj", "model.layers.74.mlp.experts.128.up_proj", "model.layers.74.mlp.experts.129.up_proj", "model.layers.74.mlp.experts.130.up_proj", "model.layers.74.mlp.experts.131.up_proj", "model.layers.74.mlp.experts.132.up_proj", "model.layers.74.mlp.experts.133.up_proj", "model.layers.74.mlp.experts.134.up_proj", "model.layers.74.mlp.experts.135.up_proj", "model.layers.74.mlp.experts.136.up_proj", "model.layers.74.mlp.experts.137.up_proj", "model.layers.74.mlp.experts.138.up_proj", "model.layers.74.mlp.experts.139.up_proj", "model.layers.74.mlp.experts.140.up_proj", "model.layers.74.mlp.experts.141.up_proj", "model.layers.74.mlp.experts.142.up_proj", "model.layers.74.mlp.experts.143.up_proj", "model.layers.74.mlp.experts.144.up_proj", "model.layers.74.mlp.experts.145.up_proj", "model.layers.74.mlp.experts.146.up_proj", "model.layers.74.mlp.experts.147.up_proj", "model.layers.74.mlp.experts.148.up_proj", "model.layers.74.mlp.experts.149.up_proj", "model.layers.74.mlp.experts.150.up_proj", "model.layers.74.mlp.experts.151.up_proj", "model.layers.74.mlp.experts.152.up_proj", "model.layers.74.mlp.experts.153.up_proj", "model.layers.74.mlp.experts.154.up_proj", "model.layers.74.mlp.experts.155.up_proj", "model.layers.74.mlp.experts.156.up_proj", "model.layers.74.mlp.experts.157.up_proj", "model.layers.74.mlp.experts.158.up_proj", "model.layers.74.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.015311323105855e-07, "dbits": 5033164800 } ] }, { "idx": 446, "layers": [ "model.layers.74.mlp.experts.0.down_proj", "model.layers.74.mlp.experts.1.down_proj", "model.layers.74.mlp.experts.2.down_proj", "model.layers.74.mlp.experts.3.down_proj", "model.layers.74.mlp.experts.4.down_proj", "model.layers.74.mlp.experts.5.down_proj", "model.layers.74.mlp.experts.6.down_proj", "model.layers.74.mlp.experts.7.down_proj", "model.layers.74.mlp.experts.8.down_proj", "model.layers.74.mlp.experts.9.down_proj", "model.layers.74.mlp.experts.10.down_proj", "model.layers.74.mlp.experts.11.down_proj", "model.layers.74.mlp.experts.12.down_proj", "model.layers.74.mlp.experts.13.down_proj", "model.layers.74.mlp.experts.14.down_proj", "model.layers.74.mlp.experts.15.down_proj", "model.layers.74.mlp.experts.16.down_proj", "model.layers.74.mlp.experts.17.down_proj", "model.layers.74.mlp.experts.18.down_proj", "model.layers.74.mlp.experts.19.down_proj", "model.layers.74.mlp.experts.20.down_proj", "model.layers.74.mlp.experts.21.down_proj", "model.layers.74.mlp.experts.22.down_proj", "model.layers.74.mlp.experts.23.down_proj", "model.layers.74.mlp.experts.24.down_proj", "model.layers.74.mlp.experts.25.down_proj", "model.layers.74.mlp.experts.26.down_proj", "model.layers.74.mlp.experts.27.down_proj", "model.layers.74.mlp.experts.28.down_proj", "model.layers.74.mlp.experts.29.down_proj", "model.layers.74.mlp.experts.30.down_proj", "model.layers.74.mlp.experts.31.down_proj", "model.layers.74.mlp.experts.32.down_proj", "model.layers.74.mlp.experts.33.down_proj", "model.layers.74.mlp.experts.34.down_proj", "model.layers.74.mlp.experts.35.down_proj", "model.layers.74.mlp.experts.36.down_proj", "model.layers.74.mlp.experts.37.down_proj", "model.layers.74.mlp.experts.38.down_proj", "model.layers.74.mlp.experts.39.down_proj", "model.layers.74.mlp.experts.40.down_proj", "model.layers.74.mlp.experts.41.down_proj", "model.layers.74.mlp.experts.42.down_proj", "model.layers.74.mlp.experts.43.down_proj", "model.layers.74.mlp.experts.44.down_proj", "model.layers.74.mlp.experts.45.down_proj", "model.layers.74.mlp.experts.46.down_proj", "model.layers.74.mlp.experts.47.down_proj", "model.layers.74.mlp.experts.48.down_proj", "model.layers.74.mlp.experts.49.down_proj", "model.layers.74.mlp.experts.50.down_proj", "model.layers.74.mlp.experts.51.down_proj", "model.layers.74.mlp.experts.52.down_proj", "model.layers.74.mlp.experts.53.down_proj", "model.layers.74.mlp.experts.54.down_proj", "model.layers.74.mlp.experts.55.down_proj", "model.layers.74.mlp.experts.56.down_proj", "model.layers.74.mlp.experts.57.down_proj", "model.layers.74.mlp.experts.58.down_proj", "model.layers.74.mlp.experts.59.down_proj", "model.layers.74.mlp.experts.60.down_proj", "model.layers.74.mlp.experts.61.down_proj", "model.layers.74.mlp.experts.62.down_proj", "model.layers.74.mlp.experts.63.down_proj", "model.layers.74.mlp.experts.64.down_proj", "model.layers.74.mlp.experts.65.down_proj", "model.layers.74.mlp.experts.66.down_proj", "model.layers.74.mlp.experts.67.down_proj", "model.layers.74.mlp.experts.68.down_proj", "model.layers.74.mlp.experts.69.down_proj", "model.layers.74.mlp.experts.70.down_proj", "model.layers.74.mlp.experts.71.down_proj", "model.layers.74.mlp.experts.72.down_proj", "model.layers.74.mlp.experts.73.down_proj", "model.layers.74.mlp.experts.74.down_proj", "model.layers.74.mlp.experts.75.down_proj", "model.layers.74.mlp.experts.76.down_proj", "model.layers.74.mlp.experts.77.down_proj", "model.layers.74.mlp.experts.78.down_proj", "model.layers.74.mlp.experts.79.down_proj", "model.layers.74.mlp.experts.80.down_proj", "model.layers.74.mlp.experts.81.down_proj", "model.layers.74.mlp.experts.82.down_proj", "model.layers.74.mlp.experts.83.down_proj", "model.layers.74.mlp.experts.84.down_proj", "model.layers.74.mlp.experts.85.down_proj", "model.layers.74.mlp.experts.86.down_proj", "model.layers.74.mlp.experts.87.down_proj", "model.layers.74.mlp.experts.88.down_proj", "model.layers.74.mlp.experts.89.down_proj", "model.layers.74.mlp.experts.90.down_proj", "model.layers.74.mlp.experts.91.down_proj", "model.layers.74.mlp.experts.92.down_proj", "model.layers.74.mlp.experts.93.down_proj", "model.layers.74.mlp.experts.94.down_proj", "model.layers.74.mlp.experts.95.down_proj", "model.layers.74.mlp.experts.96.down_proj", "model.layers.74.mlp.experts.97.down_proj", "model.layers.74.mlp.experts.98.down_proj", "model.layers.74.mlp.experts.99.down_proj", "model.layers.74.mlp.experts.100.down_proj", "model.layers.74.mlp.experts.101.down_proj", "model.layers.74.mlp.experts.102.down_proj", "model.layers.74.mlp.experts.103.down_proj", "model.layers.74.mlp.experts.104.down_proj", "model.layers.74.mlp.experts.105.down_proj", "model.layers.74.mlp.experts.106.down_proj", "model.layers.74.mlp.experts.107.down_proj", "model.layers.74.mlp.experts.108.down_proj", "model.layers.74.mlp.experts.109.down_proj", "model.layers.74.mlp.experts.110.down_proj", "model.layers.74.mlp.experts.111.down_proj", "model.layers.74.mlp.experts.112.down_proj", "model.layers.74.mlp.experts.113.down_proj", "model.layers.74.mlp.experts.114.down_proj", "model.layers.74.mlp.experts.115.down_proj", "model.layers.74.mlp.experts.116.down_proj", "model.layers.74.mlp.experts.117.down_proj", "model.layers.74.mlp.experts.118.down_proj", "model.layers.74.mlp.experts.119.down_proj", "model.layers.74.mlp.experts.120.down_proj", "model.layers.74.mlp.experts.121.down_proj", "model.layers.74.mlp.experts.122.down_proj", "model.layers.74.mlp.experts.123.down_proj", "model.layers.74.mlp.experts.124.down_proj", "model.layers.74.mlp.experts.125.down_proj", "model.layers.74.mlp.experts.126.down_proj", "model.layers.74.mlp.experts.127.down_proj", "model.layers.74.mlp.experts.128.down_proj", "model.layers.74.mlp.experts.129.down_proj", "model.layers.74.mlp.experts.130.down_proj", "model.layers.74.mlp.experts.131.down_proj", "model.layers.74.mlp.experts.132.down_proj", "model.layers.74.mlp.experts.133.down_proj", "model.layers.74.mlp.experts.134.down_proj", "model.layers.74.mlp.experts.135.down_proj", "model.layers.74.mlp.experts.136.down_proj", "model.layers.74.mlp.experts.137.down_proj", "model.layers.74.mlp.experts.138.down_proj", "model.layers.74.mlp.experts.139.down_proj", "model.layers.74.mlp.experts.140.down_proj", "model.layers.74.mlp.experts.141.down_proj", "model.layers.74.mlp.experts.142.down_proj", "model.layers.74.mlp.experts.143.down_proj", "model.layers.74.mlp.experts.144.down_proj", "model.layers.74.mlp.experts.145.down_proj", "model.layers.74.mlp.experts.146.down_proj", "model.layers.74.mlp.experts.147.down_proj", "model.layers.74.mlp.experts.148.down_proj", "model.layers.74.mlp.experts.149.down_proj", "model.layers.74.mlp.experts.150.down_proj", "model.layers.74.mlp.experts.151.down_proj", "model.layers.74.mlp.experts.152.down_proj", "model.layers.74.mlp.experts.153.down_proj", "model.layers.74.mlp.experts.154.down_proj", "model.layers.74.mlp.experts.155.down_proj", "model.layers.74.mlp.experts.156.down_proj", "model.layers.74.mlp.experts.157.down_proj", "model.layers.74.mlp.experts.158.down_proj", "model.layers.74.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -1.6117759514599822e-06, "dbits": 2516582400 } ] }, { "idx": 447, "layers": [ "model.layers.75.self_attn.q_proj" ], "candidates": [ { "dkld": 3.5904377000406384e-06, "dbits": 125829120 } ] }, { "idx": 448, "layers": [ "model.layers.75.self_attn.k_proj", "model.layers.75.self_attn.v_proj" ], "candidates": [ { "dkld": 1.520654768683038e-06, "dbits": 20971520 } ] }, { "idx": 449, "layers": [ "model.layers.75.self_attn.o_proj" ], "candidates": [ { "dkld": 3.021152224391699e-06, "dbits": 125829120 } ] }, { "idx": 450, "layers": [ "model.layers.75.mlp.shared_experts.gate_proj", "model.layers.75.mlp.shared_experts.up_proj", "model.layers.75.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.1273153834044933e-06, "dbits": 47185920 } ] }, { "idx": 451, "layers": [ "model.layers.75.mlp.experts.0.gate_proj", "model.layers.75.mlp.experts.1.gate_proj", "model.layers.75.mlp.experts.2.gate_proj", "model.layers.75.mlp.experts.3.gate_proj", "model.layers.75.mlp.experts.4.gate_proj", "model.layers.75.mlp.experts.5.gate_proj", "model.layers.75.mlp.experts.6.gate_proj", "model.layers.75.mlp.experts.7.gate_proj", "model.layers.75.mlp.experts.8.gate_proj", "model.layers.75.mlp.experts.9.gate_proj", "model.layers.75.mlp.experts.10.gate_proj", "model.layers.75.mlp.experts.11.gate_proj", "model.layers.75.mlp.experts.12.gate_proj", "model.layers.75.mlp.experts.13.gate_proj", "model.layers.75.mlp.experts.14.gate_proj", "model.layers.75.mlp.experts.15.gate_proj", "model.layers.75.mlp.experts.16.gate_proj", "model.layers.75.mlp.experts.17.gate_proj", "model.layers.75.mlp.experts.18.gate_proj", "model.layers.75.mlp.experts.19.gate_proj", "model.layers.75.mlp.experts.20.gate_proj", "model.layers.75.mlp.experts.21.gate_proj", "model.layers.75.mlp.experts.22.gate_proj", "model.layers.75.mlp.experts.23.gate_proj", "model.layers.75.mlp.experts.24.gate_proj", "model.layers.75.mlp.experts.25.gate_proj", "model.layers.75.mlp.experts.26.gate_proj", "model.layers.75.mlp.experts.27.gate_proj", "model.layers.75.mlp.experts.28.gate_proj", "model.layers.75.mlp.experts.29.gate_proj", "model.layers.75.mlp.experts.30.gate_proj", "model.layers.75.mlp.experts.31.gate_proj", "model.layers.75.mlp.experts.32.gate_proj", "model.layers.75.mlp.experts.33.gate_proj", "model.layers.75.mlp.experts.34.gate_proj", "model.layers.75.mlp.experts.35.gate_proj", "model.layers.75.mlp.experts.36.gate_proj", "model.layers.75.mlp.experts.37.gate_proj", "model.layers.75.mlp.experts.38.gate_proj", "model.layers.75.mlp.experts.39.gate_proj", "model.layers.75.mlp.experts.40.gate_proj", "model.layers.75.mlp.experts.41.gate_proj", "model.layers.75.mlp.experts.42.gate_proj", "model.layers.75.mlp.experts.43.gate_proj", "model.layers.75.mlp.experts.44.gate_proj", "model.layers.75.mlp.experts.45.gate_proj", "model.layers.75.mlp.experts.46.gate_proj", "model.layers.75.mlp.experts.47.gate_proj", "model.layers.75.mlp.experts.48.gate_proj", "model.layers.75.mlp.experts.49.gate_proj", "model.layers.75.mlp.experts.50.gate_proj", "model.layers.75.mlp.experts.51.gate_proj", "model.layers.75.mlp.experts.52.gate_proj", "model.layers.75.mlp.experts.53.gate_proj", "model.layers.75.mlp.experts.54.gate_proj", "model.layers.75.mlp.experts.55.gate_proj", "model.layers.75.mlp.experts.56.gate_proj", "model.layers.75.mlp.experts.57.gate_proj", "model.layers.75.mlp.experts.58.gate_proj", "model.layers.75.mlp.experts.59.gate_proj", "model.layers.75.mlp.experts.60.gate_proj", "model.layers.75.mlp.experts.61.gate_proj", "model.layers.75.mlp.experts.62.gate_proj", "model.layers.75.mlp.experts.63.gate_proj", "model.layers.75.mlp.experts.64.gate_proj", "model.layers.75.mlp.experts.65.gate_proj", "model.layers.75.mlp.experts.66.gate_proj", "model.layers.75.mlp.experts.67.gate_proj", "model.layers.75.mlp.experts.68.gate_proj", "model.layers.75.mlp.experts.69.gate_proj", "model.layers.75.mlp.experts.70.gate_proj", "model.layers.75.mlp.experts.71.gate_proj", "model.layers.75.mlp.experts.72.gate_proj", "model.layers.75.mlp.experts.73.gate_proj", "model.layers.75.mlp.experts.74.gate_proj", "model.layers.75.mlp.experts.75.gate_proj", "model.layers.75.mlp.experts.76.gate_proj", "model.layers.75.mlp.experts.77.gate_proj", "model.layers.75.mlp.experts.78.gate_proj", "model.layers.75.mlp.experts.79.gate_proj", "model.layers.75.mlp.experts.80.gate_proj", "model.layers.75.mlp.experts.81.gate_proj", "model.layers.75.mlp.experts.82.gate_proj", "model.layers.75.mlp.experts.83.gate_proj", "model.layers.75.mlp.experts.84.gate_proj", "model.layers.75.mlp.experts.85.gate_proj", "model.layers.75.mlp.experts.86.gate_proj", "model.layers.75.mlp.experts.87.gate_proj", "model.layers.75.mlp.experts.88.gate_proj", "model.layers.75.mlp.experts.89.gate_proj", "model.layers.75.mlp.experts.90.gate_proj", "model.layers.75.mlp.experts.91.gate_proj", "model.layers.75.mlp.experts.92.gate_proj", "model.layers.75.mlp.experts.93.gate_proj", "model.layers.75.mlp.experts.94.gate_proj", "model.layers.75.mlp.experts.95.gate_proj", "model.layers.75.mlp.experts.96.gate_proj", "model.layers.75.mlp.experts.97.gate_proj", "model.layers.75.mlp.experts.98.gate_proj", "model.layers.75.mlp.experts.99.gate_proj", "model.layers.75.mlp.experts.100.gate_proj", "model.layers.75.mlp.experts.101.gate_proj", "model.layers.75.mlp.experts.102.gate_proj", "model.layers.75.mlp.experts.103.gate_proj", "model.layers.75.mlp.experts.104.gate_proj", "model.layers.75.mlp.experts.105.gate_proj", "model.layers.75.mlp.experts.106.gate_proj", "model.layers.75.mlp.experts.107.gate_proj", "model.layers.75.mlp.experts.108.gate_proj", "model.layers.75.mlp.experts.109.gate_proj", "model.layers.75.mlp.experts.110.gate_proj", "model.layers.75.mlp.experts.111.gate_proj", "model.layers.75.mlp.experts.112.gate_proj", "model.layers.75.mlp.experts.113.gate_proj", "model.layers.75.mlp.experts.114.gate_proj", "model.layers.75.mlp.experts.115.gate_proj", "model.layers.75.mlp.experts.116.gate_proj", "model.layers.75.mlp.experts.117.gate_proj", "model.layers.75.mlp.experts.118.gate_proj", "model.layers.75.mlp.experts.119.gate_proj", "model.layers.75.mlp.experts.120.gate_proj", "model.layers.75.mlp.experts.121.gate_proj", "model.layers.75.mlp.experts.122.gate_proj", "model.layers.75.mlp.experts.123.gate_proj", "model.layers.75.mlp.experts.124.gate_proj", "model.layers.75.mlp.experts.125.gate_proj", "model.layers.75.mlp.experts.126.gate_proj", "model.layers.75.mlp.experts.127.gate_proj", "model.layers.75.mlp.experts.128.gate_proj", "model.layers.75.mlp.experts.129.gate_proj", "model.layers.75.mlp.experts.130.gate_proj", "model.layers.75.mlp.experts.131.gate_proj", "model.layers.75.mlp.experts.132.gate_proj", "model.layers.75.mlp.experts.133.gate_proj", "model.layers.75.mlp.experts.134.gate_proj", "model.layers.75.mlp.experts.135.gate_proj", "model.layers.75.mlp.experts.136.gate_proj", "model.layers.75.mlp.experts.137.gate_proj", "model.layers.75.mlp.experts.138.gate_proj", "model.layers.75.mlp.experts.139.gate_proj", "model.layers.75.mlp.experts.140.gate_proj", "model.layers.75.mlp.experts.141.gate_proj", "model.layers.75.mlp.experts.142.gate_proj", "model.layers.75.mlp.experts.143.gate_proj", "model.layers.75.mlp.experts.144.gate_proj", "model.layers.75.mlp.experts.145.gate_proj", "model.layers.75.mlp.experts.146.gate_proj", "model.layers.75.mlp.experts.147.gate_proj", "model.layers.75.mlp.experts.148.gate_proj", "model.layers.75.mlp.experts.149.gate_proj", "model.layers.75.mlp.experts.150.gate_proj", "model.layers.75.mlp.experts.151.gate_proj", "model.layers.75.mlp.experts.152.gate_proj", "model.layers.75.mlp.experts.153.gate_proj", "model.layers.75.mlp.experts.154.gate_proj", "model.layers.75.mlp.experts.155.gate_proj", "model.layers.75.mlp.experts.156.gate_proj", "model.layers.75.mlp.experts.157.gate_proj", "model.layers.75.mlp.experts.158.gate_proj", "model.layers.75.mlp.experts.159.gate_proj", "model.layers.75.mlp.experts.0.up_proj", "model.layers.75.mlp.experts.1.up_proj", "model.layers.75.mlp.experts.2.up_proj", "model.layers.75.mlp.experts.3.up_proj", "model.layers.75.mlp.experts.4.up_proj", "model.layers.75.mlp.experts.5.up_proj", "model.layers.75.mlp.experts.6.up_proj", "model.layers.75.mlp.experts.7.up_proj", "model.layers.75.mlp.experts.8.up_proj", "model.layers.75.mlp.experts.9.up_proj", "model.layers.75.mlp.experts.10.up_proj", "model.layers.75.mlp.experts.11.up_proj", "model.layers.75.mlp.experts.12.up_proj", "model.layers.75.mlp.experts.13.up_proj", "model.layers.75.mlp.experts.14.up_proj", "model.layers.75.mlp.experts.15.up_proj", "model.layers.75.mlp.experts.16.up_proj", "model.layers.75.mlp.experts.17.up_proj", "model.layers.75.mlp.experts.18.up_proj", "model.layers.75.mlp.experts.19.up_proj", "model.layers.75.mlp.experts.20.up_proj", "model.layers.75.mlp.experts.21.up_proj", "model.layers.75.mlp.experts.22.up_proj", "model.layers.75.mlp.experts.23.up_proj", "model.layers.75.mlp.experts.24.up_proj", "model.layers.75.mlp.experts.25.up_proj", "model.layers.75.mlp.experts.26.up_proj", "model.layers.75.mlp.experts.27.up_proj", "model.layers.75.mlp.experts.28.up_proj", "model.layers.75.mlp.experts.29.up_proj", "model.layers.75.mlp.experts.30.up_proj", "model.layers.75.mlp.experts.31.up_proj", "model.layers.75.mlp.experts.32.up_proj", "model.layers.75.mlp.experts.33.up_proj", "model.layers.75.mlp.experts.34.up_proj", "model.layers.75.mlp.experts.35.up_proj", "model.layers.75.mlp.experts.36.up_proj", "model.layers.75.mlp.experts.37.up_proj", "model.layers.75.mlp.experts.38.up_proj", "model.layers.75.mlp.experts.39.up_proj", "model.layers.75.mlp.experts.40.up_proj", "model.layers.75.mlp.experts.41.up_proj", "model.layers.75.mlp.experts.42.up_proj", "model.layers.75.mlp.experts.43.up_proj", "model.layers.75.mlp.experts.44.up_proj", "model.layers.75.mlp.experts.45.up_proj", "model.layers.75.mlp.experts.46.up_proj", "model.layers.75.mlp.experts.47.up_proj", "model.layers.75.mlp.experts.48.up_proj", "model.layers.75.mlp.experts.49.up_proj", "model.layers.75.mlp.experts.50.up_proj", "model.layers.75.mlp.experts.51.up_proj", "model.layers.75.mlp.experts.52.up_proj", "model.layers.75.mlp.experts.53.up_proj", "model.layers.75.mlp.experts.54.up_proj", "model.layers.75.mlp.experts.55.up_proj", "model.layers.75.mlp.experts.56.up_proj", "model.layers.75.mlp.experts.57.up_proj", "model.layers.75.mlp.experts.58.up_proj", "model.layers.75.mlp.experts.59.up_proj", "model.layers.75.mlp.experts.60.up_proj", "model.layers.75.mlp.experts.61.up_proj", "model.layers.75.mlp.experts.62.up_proj", "model.layers.75.mlp.experts.63.up_proj", "model.layers.75.mlp.experts.64.up_proj", "model.layers.75.mlp.experts.65.up_proj", "model.layers.75.mlp.experts.66.up_proj", "model.layers.75.mlp.experts.67.up_proj", "model.layers.75.mlp.experts.68.up_proj", "model.layers.75.mlp.experts.69.up_proj", "model.layers.75.mlp.experts.70.up_proj", "model.layers.75.mlp.experts.71.up_proj", "model.layers.75.mlp.experts.72.up_proj", "model.layers.75.mlp.experts.73.up_proj", "model.layers.75.mlp.experts.74.up_proj", "model.layers.75.mlp.experts.75.up_proj", "model.layers.75.mlp.experts.76.up_proj", "model.layers.75.mlp.experts.77.up_proj", "model.layers.75.mlp.experts.78.up_proj", "model.layers.75.mlp.experts.79.up_proj", "model.layers.75.mlp.experts.80.up_proj", "model.layers.75.mlp.experts.81.up_proj", "model.layers.75.mlp.experts.82.up_proj", "model.layers.75.mlp.experts.83.up_proj", "model.layers.75.mlp.experts.84.up_proj", "model.layers.75.mlp.experts.85.up_proj", "model.layers.75.mlp.experts.86.up_proj", "model.layers.75.mlp.experts.87.up_proj", "model.layers.75.mlp.experts.88.up_proj", "model.layers.75.mlp.experts.89.up_proj", "model.layers.75.mlp.experts.90.up_proj", "model.layers.75.mlp.experts.91.up_proj", "model.layers.75.mlp.experts.92.up_proj", "model.layers.75.mlp.experts.93.up_proj", "model.layers.75.mlp.experts.94.up_proj", "model.layers.75.mlp.experts.95.up_proj", "model.layers.75.mlp.experts.96.up_proj", "model.layers.75.mlp.experts.97.up_proj", "model.layers.75.mlp.experts.98.up_proj", "model.layers.75.mlp.experts.99.up_proj", "model.layers.75.mlp.experts.100.up_proj", "model.layers.75.mlp.experts.101.up_proj", "model.layers.75.mlp.experts.102.up_proj", "model.layers.75.mlp.experts.103.up_proj", "model.layers.75.mlp.experts.104.up_proj", "model.layers.75.mlp.experts.105.up_proj", "model.layers.75.mlp.experts.106.up_proj", "model.layers.75.mlp.experts.107.up_proj", "model.layers.75.mlp.experts.108.up_proj", "model.layers.75.mlp.experts.109.up_proj", "model.layers.75.mlp.experts.110.up_proj", "model.layers.75.mlp.experts.111.up_proj", "model.layers.75.mlp.experts.112.up_proj", "model.layers.75.mlp.experts.113.up_proj", "model.layers.75.mlp.experts.114.up_proj", "model.layers.75.mlp.experts.115.up_proj", "model.layers.75.mlp.experts.116.up_proj", "model.layers.75.mlp.experts.117.up_proj", "model.layers.75.mlp.experts.118.up_proj", "model.layers.75.mlp.experts.119.up_proj", "model.layers.75.mlp.experts.120.up_proj", "model.layers.75.mlp.experts.121.up_proj", "model.layers.75.mlp.experts.122.up_proj", "model.layers.75.mlp.experts.123.up_proj", "model.layers.75.mlp.experts.124.up_proj", "model.layers.75.mlp.experts.125.up_proj", "model.layers.75.mlp.experts.126.up_proj", "model.layers.75.mlp.experts.127.up_proj", "model.layers.75.mlp.experts.128.up_proj", "model.layers.75.mlp.experts.129.up_proj", "model.layers.75.mlp.experts.130.up_proj", "model.layers.75.mlp.experts.131.up_proj", "model.layers.75.mlp.experts.132.up_proj", "model.layers.75.mlp.experts.133.up_proj", "model.layers.75.mlp.experts.134.up_proj", "model.layers.75.mlp.experts.135.up_proj", "model.layers.75.mlp.experts.136.up_proj", "model.layers.75.mlp.experts.137.up_proj", "model.layers.75.mlp.experts.138.up_proj", "model.layers.75.mlp.experts.139.up_proj", "model.layers.75.mlp.experts.140.up_proj", "model.layers.75.mlp.experts.141.up_proj", "model.layers.75.mlp.experts.142.up_proj", "model.layers.75.mlp.experts.143.up_proj", "model.layers.75.mlp.experts.144.up_proj", "model.layers.75.mlp.experts.145.up_proj", "model.layers.75.mlp.experts.146.up_proj", "model.layers.75.mlp.experts.147.up_proj", "model.layers.75.mlp.experts.148.up_proj", "model.layers.75.mlp.experts.149.up_proj", "model.layers.75.mlp.experts.150.up_proj", "model.layers.75.mlp.experts.151.up_proj", "model.layers.75.mlp.experts.152.up_proj", "model.layers.75.mlp.experts.153.up_proj", "model.layers.75.mlp.experts.154.up_proj", "model.layers.75.mlp.experts.155.up_proj", "model.layers.75.mlp.experts.156.up_proj", "model.layers.75.mlp.experts.157.up_proj", "model.layers.75.mlp.experts.158.up_proj", "model.layers.75.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.4298235075548725e-06, "dbits": 5033164800 } ] }, { "idx": 452, "layers": [ "model.layers.75.mlp.experts.0.down_proj", "model.layers.75.mlp.experts.1.down_proj", "model.layers.75.mlp.experts.2.down_proj", "model.layers.75.mlp.experts.3.down_proj", "model.layers.75.mlp.experts.4.down_proj", "model.layers.75.mlp.experts.5.down_proj", "model.layers.75.mlp.experts.6.down_proj", "model.layers.75.mlp.experts.7.down_proj", "model.layers.75.mlp.experts.8.down_proj", "model.layers.75.mlp.experts.9.down_proj", "model.layers.75.mlp.experts.10.down_proj", "model.layers.75.mlp.experts.11.down_proj", "model.layers.75.mlp.experts.12.down_proj", "model.layers.75.mlp.experts.13.down_proj", "model.layers.75.mlp.experts.14.down_proj", "model.layers.75.mlp.experts.15.down_proj", "model.layers.75.mlp.experts.16.down_proj", "model.layers.75.mlp.experts.17.down_proj", "model.layers.75.mlp.experts.18.down_proj", "model.layers.75.mlp.experts.19.down_proj", "model.layers.75.mlp.experts.20.down_proj", "model.layers.75.mlp.experts.21.down_proj", "model.layers.75.mlp.experts.22.down_proj", "model.layers.75.mlp.experts.23.down_proj", "model.layers.75.mlp.experts.24.down_proj", "model.layers.75.mlp.experts.25.down_proj", "model.layers.75.mlp.experts.26.down_proj", "model.layers.75.mlp.experts.27.down_proj", "model.layers.75.mlp.experts.28.down_proj", "model.layers.75.mlp.experts.29.down_proj", "model.layers.75.mlp.experts.30.down_proj", "model.layers.75.mlp.experts.31.down_proj", "model.layers.75.mlp.experts.32.down_proj", "model.layers.75.mlp.experts.33.down_proj", "model.layers.75.mlp.experts.34.down_proj", "model.layers.75.mlp.experts.35.down_proj", "model.layers.75.mlp.experts.36.down_proj", "model.layers.75.mlp.experts.37.down_proj", "model.layers.75.mlp.experts.38.down_proj", "model.layers.75.mlp.experts.39.down_proj", "model.layers.75.mlp.experts.40.down_proj", "model.layers.75.mlp.experts.41.down_proj", "model.layers.75.mlp.experts.42.down_proj", "model.layers.75.mlp.experts.43.down_proj", "model.layers.75.mlp.experts.44.down_proj", "model.layers.75.mlp.experts.45.down_proj", "model.layers.75.mlp.experts.46.down_proj", "model.layers.75.mlp.experts.47.down_proj", "model.layers.75.mlp.experts.48.down_proj", "model.layers.75.mlp.experts.49.down_proj", "model.layers.75.mlp.experts.50.down_proj", "model.layers.75.mlp.experts.51.down_proj", "model.layers.75.mlp.experts.52.down_proj", "model.layers.75.mlp.experts.53.down_proj", "model.layers.75.mlp.experts.54.down_proj", "model.layers.75.mlp.experts.55.down_proj", "model.layers.75.mlp.experts.56.down_proj", "model.layers.75.mlp.experts.57.down_proj", "model.layers.75.mlp.experts.58.down_proj", "model.layers.75.mlp.experts.59.down_proj", "model.layers.75.mlp.experts.60.down_proj", "model.layers.75.mlp.experts.61.down_proj", "model.layers.75.mlp.experts.62.down_proj", "model.layers.75.mlp.experts.63.down_proj", "model.layers.75.mlp.experts.64.down_proj", "model.layers.75.mlp.experts.65.down_proj", "model.layers.75.mlp.experts.66.down_proj", "model.layers.75.mlp.experts.67.down_proj", "model.layers.75.mlp.experts.68.down_proj", "model.layers.75.mlp.experts.69.down_proj", "model.layers.75.mlp.experts.70.down_proj", "model.layers.75.mlp.experts.71.down_proj", "model.layers.75.mlp.experts.72.down_proj", "model.layers.75.mlp.experts.73.down_proj", "model.layers.75.mlp.experts.74.down_proj", "model.layers.75.mlp.experts.75.down_proj", "model.layers.75.mlp.experts.76.down_proj", "model.layers.75.mlp.experts.77.down_proj", "model.layers.75.mlp.experts.78.down_proj", "model.layers.75.mlp.experts.79.down_proj", "model.layers.75.mlp.experts.80.down_proj", "model.layers.75.mlp.experts.81.down_proj", "model.layers.75.mlp.experts.82.down_proj", "model.layers.75.mlp.experts.83.down_proj", "model.layers.75.mlp.experts.84.down_proj", "model.layers.75.mlp.experts.85.down_proj", "model.layers.75.mlp.experts.86.down_proj", "model.layers.75.mlp.experts.87.down_proj", "model.layers.75.mlp.experts.88.down_proj", "model.layers.75.mlp.experts.89.down_proj", "model.layers.75.mlp.experts.90.down_proj", "model.layers.75.mlp.experts.91.down_proj", "model.layers.75.mlp.experts.92.down_proj", "model.layers.75.mlp.experts.93.down_proj", "model.layers.75.mlp.experts.94.down_proj", "model.layers.75.mlp.experts.95.down_proj", "model.layers.75.mlp.experts.96.down_proj", "model.layers.75.mlp.experts.97.down_proj", "model.layers.75.mlp.experts.98.down_proj", "model.layers.75.mlp.experts.99.down_proj", "model.layers.75.mlp.experts.100.down_proj", "model.layers.75.mlp.experts.101.down_proj", "model.layers.75.mlp.experts.102.down_proj", "model.layers.75.mlp.experts.103.down_proj", "model.layers.75.mlp.experts.104.down_proj", "model.layers.75.mlp.experts.105.down_proj", "model.layers.75.mlp.experts.106.down_proj", "model.layers.75.mlp.experts.107.down_proj", "model.layers.75.mlp.experts.108.down_proj", "model.layers.75.mlp.experts.109.down_proj", "model.layers.75.mlp.experts.110.down_proj", "model.layers.75.mlp.experts.111.down_proj", "model.layers.75.mlp.experts.112.down_proj", "model.layers.75.mlp.experts.113.down_proj", "model.layers.75.mlp.experts.114.down_proj", "model.layers.75.mlp.experts.115.down_proj", "model.layers.75.mlp.experts.116.down_proj", "model.layers.75.mlp.experts.117.down_proj", "model.layers.75.mlp.experts.118.down_proj", "model.layers.75.mlp.experts.119.down_proj", "model.layers.75.mlp.experts.120.down_proj", "model.layers.75.mlp.experts.121.down_proj", "model.layers.75.mlp.experts.122.down_proj", "model.layers.75.mlp.experts.123.down_proj", "model.layers.75.mlp.experts.124.down_proj", "model.layers.75.mlp.experts.125.down_proj", "model.layers.75.mlp.experts.126.down_proj", "model.layers.75.mlp.experts.127.down_proj", "model.layers.75.mlp.experts.128.down_proj", "model.layers.75.mlp.experts.129.down_proj", "model.layers.75.mlp.experts.130.down_proj", "model.layers.75.mlp.experts.131.down_proj", "model.layers.75.mlp.experts.132.down_proj", "model.layers.75.mlp.experts.133.down_proj", "model.layers.75.mlp.experts.134.down_proj", "model.layers.75.mlp.experts.135.down_proj", "model.layers.75.mlp.experts.136.down_proj", "model.layers.75.mlp.experts.137.down_proj", "model.layers.75.mlp.experts.138.down_proj", "model.layers.75.mlp.experts.139.down_proj", "model.layers.75.mlp.experts.140.down_proj", "model.layers.75.mlp.experts.141.down_proj", "model.layers.75.mlp.experts.142.down_proj", "model.layers.75.mlp.experts.143.down_proj", "model.layers.75.mlp.experts.144.down_proj", "model.layers.75.mlp.experts.145.down_proj", "model.layers.75.mlp.experts.146.down_proj", "model.layers.75.mlp.experts.147.down_proj", "model.layers.75.mlp.experts.148.down_proj", "model.layers.75.mlp.experts.149.down_proj", "model.layers.75.mlp.experts.150.down_proj", "model.layers.75.mlp.experts.151.down_proj", "model.layers.75.mlp.experts.152.down_proj", "model.layers.75.mlp.experts.153.down_proj", "model.layers.75.mlp.experts.154.down_proj", "model.layers.75.mlp.experts.155.down_proj", "model.layers.75.mlp.experts.156.down_proj", "model.layers.75.mlp.experts.157.down_proj", "model.layers.75.mlp.experts.158.down_proj", "model.layers.75.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.285575076006353e-06, "dbits": 2516582400 } ] }, { "idx": 453, "layers": [ "model.layers.76.self_attn.q_proj" ], "candidates": [ { "dkld": 4.3733482016250925e-06, "dbits": 125829120 } ] }, { "idx": 454, "layers": [ "model.layers.76.self_attn.k_proj", "model.layers.76.self_attn.v_proj" ], "candidates": [ { "dkld": 3.113638376817129e-06, "dbits": 20971520 } ] }, { "idx": 455, "layers": [ "model.layers.76.self_attn.o_proj" ], "candidates": [ { "dkld": 3.90727655030787e-06, "dbits": 125829120 } ] }, { "idx": 456, "layers": [ "model.layers.76.mlp.shared_experts.gate_proj", "model.layers.76.mlp.shared_experts.up_proj", "model.layers.76.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.460583345964551e-07, "dbits": 47185920 } ] }, { "idx": 457, "layers": [ "model.layers.76.mlp.experts.0.gate_proj", "model.layers.76.mlp.experts.1.gate_proj", "model.layers.76.mlp.experts.2.gate_proj", "model.layers.76.mlp.experts.3.gate_proj", "model.layers.76.mlp.experts.4.gate_proj", "model.layers.76.mlp.experts.5.gate_proj", "model.layers.76.mlp.experts.6.gate_proj", "model.layers.76.mlp.experts.7.gate_proj", "model.layers.76.mlp.experts.8.gate_proj", "model.layers.76.mlp.experts.9.gate_proj", "model.layers.76.mlp.experts.10.gate_proj", "model.layers.76.mlp.experts.11.gate_proj", "model.layers.76.mlp.experts.12.gate_proj", "model.layers.76.mlp.experts.13.gate_proj", "model.layers.76.mlp.experts.14.gate_proj", "model.layers.76.mlp.experts.15.gate_proj", "model.layers.76.mlp.experts.16.gate_proj", "model.layers.76.mlp.experts.17.gate_proj", "model.layers.76.mlp.experts.18.gate_proj", "model.layers.76.mlp.experts.19.gate_proj", "model.layers.76.mlp.experts.20.gate_proj", "model.layers.76.mlp.experts.21.gate_proj", "model.layers.76.mlp.experts.22.gate_proj", "model.layers.76.mlp.experts.23.gate_proj", "model.layers.76.mlp.experts.24.gate_proj", "model.layers.76.mlp.experts.25.gate_proj", "model.layers.76.mlp.experts.26.gate_proj", "model.layers.76.mlp.experts.27.gate_proj", "model.layers.76.mlp.experts.28.gate_proj", "model.layers.76.mlp.experts.29.gate_proj", "model.layers.76.mlp.experts.30.gate_proj", "model.layers.76.mlp.experts.31.gate_proj", "model.layers.76.mlp.experts.32.gate_proj", "model.layers.76.mlp.experts.33.gate_proj", "model.layers.76.mlp.experts.34.gate_proj", "model.layers.76.mlp.experts.35.gate_proj", "model.layers.76.mlp.experts.36.gate_proj", "model.layers.76.mlp.experts.37.gate_proj", "model.layers.76.mlp.experts.38.gate_proj", "model.layers.76.mlp.experts.39.gate_proj", "model.layers.76.mlp.experts.40.gate_proj", "model.layers.76.mlp.experts.41.gate_proj", "model.layers.76.mlp.experts.42.gate_proj", "model.layers.76.mlp.experts.43.gate_proj", "model.layers.76.mlp.experts.44.gate_proj", "model.layers.76.mlp.experts.45.gate_proj", "model.layers.76.mlp.experts.46.gate_proj", "model.layers.76.mlp.experts.47.gate_proj", "model.layers.76.mlp.experts.48.gate_proj", "model.layers.76.mlp.experts.49.gate_proj", "model.layers.76.mlp.experts.50.gate_proj", "model.layers.76.mlp.experts.51.gate_proj", "model.layers.76.mlp.experts.52.gate_proj", "model.layers.76.mlp.experts.53.gate_proj", "model.layers.76.mlp.experts.54.gate_proj", "model.layers.76.mlp.experts.55.gate_proj", "model.layers.76.mlp.experts.56.gate_proj", "model.layers.76.mlp.experts.57.gate_proj", "model.layers.76.mlp.experts.58.gate_proj", "model.layers.76.mlp.experts.59.gate_proj", "model.layers.76.mlp.experts.60.gate_proj", "model.layers.76.mlp.experts.61.gate_proj", "model.layers.76.mlp.experts.62.gate_proj", "model.layers.76.mlp.experts.63.gate_proj", "model.layers.76.mlp.experts.64.gate_proj", "model.layers.76.mlp.experts.65.gate_proj", "model.layers.76.mlp.experts.66.gate_proj", "model.layers.76.mlp.experts.67.gate_proj", "model.layers.76.mlp.experts.68.gate_proj", "model.layers.76.mlp.experts.69.gate_proj", "model.layers.76.mlp.experts.70.gate_proj", "model.layers.76.mlp.experts.71.gate_proj", "model.layers.76.mlp.experts.72.gate_proj", "model.layers.76.mlp.experts.73.gate_proj", "model.layers.76.mlp.experts.74.gate_proj", "model.layers.76.mlp.experts.75.gate_proj", "model.layers.76.mlp.experts.76.gate_proj", "model.layers.76.mlp.experts.77.gate_proj", "model.layers.76.mlp.experts.78.gate_proj", "model.layers.76.mlp.experts.79.gate_proj", "model.layers.76.mlp.experts.80.gate_proj", "model.layers.76.mlp.experts.81.gate_proj", "model.layers.76.mlp.experts.82.gate_proj", "model.layers.76.mlp.experts.83.gate_proj", "model.layers.76.mlp.experts.84.gate_proj", "model.layers.76.mlp.experts.85.gate_proj", "model.layers.76.mlp.experts.86.gate_proj", "model.layers.76.mlp.experts.87.gate_proj", "model.layers.76.mlp.experts.88.gate_proj", "model.layers.76.mlp.experts.89.gate_proj", "model.layers.76.mlp.experts.90.gate_proj", "model.layers.76.mlp.experts.91.gate_proj", "model.layers.76.mlp.experts.92.gate_proj", "model.layers.76.mlp.experts.93.gate_proj", "model.layers.76.mlp.experts.94.gate_proj", "model.layers.76.mlp.experts.95.gate_proj", "model.layers.76.mlp.experts.96.gate_proj", "model.layers.76.mlp.experts.97.gate_proj", "model.layers.76.mlp.experts.98.gate_proj", "model.layers.76.mlp.experts.99.gate_proj", "model.layers.76.mlp.experts.100.gate_proj", "model.layers.76.mlp.experts.101.gate_proj", "model.layers.76.mlp.experts.102.gate_proj", "model.layers.76.mlp.experts.103.gate_proj", "model.layers.76.mlp.experts.104.gate_proj", "model.layers.76.mlp.experts.105.gate_proj", "model.layers.76.mlp.experts.106.gate_proj", "model.layers.76.mlp.experts.107.gate_proj", "model.layers.76.mlp.experts.108.gate_proj", "model.layers.76.mlp.experts.109.gate_proj", "model.layers.76.mlp.experts.110.gate_proj", "model.layers.76.mlp.experts.111.gate_proj", "model.layers.76.mlp.experts.112.gate_proj", "model.layers.76.mlp.experts.113.gate_proj", "model.layers.76.mlp.experts.114.gate_proj", "model.layers.76.mlp.experts.115.gate_proj", "model.layers.76.mlp.experts.116.gate_proj", "model.layers.76.mlp.experts.117.gate_proj", "model.layers.76.mlp.experts.118.gate_proj", "model.layers.76.mlp.experts.119.gate_proj", "model.layers.76.mlp.experts.120.gate_proj", "model.layers.76.mlp.experts.121.gate_proj", "model.layers.76.mlp.experts.122.gate_proj", "model.layers.76.mlp.experts.123.gate_proj", "model.layers.76.mlp.experts.124.gate_proj", "model.layers.76.mlp.experts.125.gate_proj", "model.layers.76.mlp.experts.126.gate_proj", "model.layers.76.mlp.experts.127.gate_proj", "model.layers.76.mlp.experts.128.gate_proj", "model.layers.76.mlp.experts.129.gate_proj", "model.layers.76.mlp.experts.130.gate_proj", "model.layers.76.mlp.experts.131.gate_proj", "model.layers.76.mlp.experts.132.gate_proj", "model.layers.76.mlp.experts.133.gate_proj", "model.layers.76.mlp.experts.134.gate_proj", "model.layers.76.mlp.experts.135.gate_proj", "model.layers.76.mlp.experts.136.gate_proj", "model.layers.76.mlp.experts.137.gate_proj", "model.layers.76.mlp.experts.138.gate_proj", "model.layers.76.mlp.experts.139.gate_proj", "model.layers.76.mlp.experts.140.gate_proj", "model.layers.76.mlp.experts.141.gate_proj", "model.layers.76.mlp.experts.142.gate_proj", "model.layers.76.mlp.experts.143.gate_proj", "model.layers.76.mlp.experts.144.gate_proj", "model.layers.76.mlp.experts.145.gate_proj", "model.layers.76.mlp.experts.146.gate_proj", "model.layers.76.mlp.experts.147.gate_proj", "model.layers.76.mlp.experts.148.gate_proj", "model.layers.76.mlp.experts.149.gate_proj", "model.layers.76.mlp.experts.150.gate_proj", "model.layers.76.mlp.experts.151.gate_proj", "model.layers.76.mlp.experts.152.gate_proj", "model.layers.76.mlp.experts.153.gate_proj", "model.layers.76.mlp.experts.154.gate_proj", "model.layers.76.mlp.experts.155.gate_proj", "model.layers.76.mlp.experts.156.gate_proj", "model.layers.76.mlp.experts.157.gate_proj", "model.layers.76.mlp.experts.158.gate_proj", "model.layers.76.mlp.experts.159.gate_proj", "model.layers.76.mlp.experts.0.up_proj", "model.layers.76.mlp.experts.1.up_proj", "model.layers.76.mlp.experts.2.up_proj", "model.layers.76.mlp.experts.3.up_proj", "model.layers.76.mlp.experts.4.up_proj", "model.layers.76.mlp.experts.5.up_proj", "model.layers.76.mlp.experts.6.up_proj", "model.layers.76.mlp.experts.7.up_proj", "model.layers.76.mlp.experts.8.up_proj", "model.layers.76.mlp.experts.9.up_proj", "model.layers.76.mlp.experts.10.up_proj", "model.layers.76.mlp.experts.11.up_proj", "model.layers.76.mlp.experts.12.up_proj", "model.layers.76.mlp.experts.13.up_proj", "model.layers.76.mlp.experts.14.up_proj", "model.layers.76.mlp.experts.15.up_proj", "model.layers.76.mlp.experts.16.up_proj", "model.layers.76.mlp.experts.17.up_proj", "model.layers.76.mlp.experts.18.up_proj", "model.layers.76.mlp.experts.19.up_proj", "model.layers.76.mlp.experts.20.up_proj", "model.layers.76.mlp.experts.21.up_proj", "model.layers.76.mlp.experts.22.up_proj", "model.layers.76.mlp.experts.23.up_proj", "model.layers.76.mlp.experts.24.up_proj", "model.layers.76.mlp.experts.25.up_proj", "model.layers.76.mlp.experts.26.up_proj", "model.layers.76.mlp.experts.27.up_proj", "model.layers.76.mlp.experts.28.up_proj", "model.layers.76.mlp.experts.29.up_proj", "model.layers.76.mlp.experts.30.up_proj", "model.layers.76.mlp.experts.31.up_proj", "model.layers.76.mlp.experts.32.up_proj", "model.layers.76.mlp.experts.33.up_proj", "model.layers.76.mlp.experts.34.up_proj", "model.layers.76.mlp.experts.35.up_proj", "model.layers.76.mlp.experts.36.up_proj", "model.layers.76.mlp.experts.37.up_proj", "model.layers.76.mlp.experts.38.up_proj", "model.layers.76.mlp.experts.39.up_proj", "model.layers.76.mlp.experts.40.up_proj", "model.layers.76.mlp.experts.41.up_proj", "model.layers.76.mlp.experts.42.up_proj", "model.layers.76.mlp.experts.43.up_proj", "model.layers.76.mlp.experts.44.up_proj", "model.layers.76.mlp.experts.45.up_proj", "model.layers.76.mlp.experts.46.up_proj", "model.layers.76.mlp.experts.47.up_proj", "model.layers.76.mlp.experts.48.up_proj", "model.layers.76.mlp.experts.49.up_proj", "model.layers.76.mlp.experts.50.up_proj", "model.layers.76.mlp.experts.51.up_proj", "model.layers.76.mlp.experts.52.up_proj", "model.layers.76.mlp.experts.53.up_proj", "model.layers.76.mlp.experts.54.up_proj", "model.layers.76.mlp.experts.55.up_proj", "model.layers.76.mlp.experts.56.up_proj", "model.layers.76.mlp.experts.57.up_proj", "model.layers.76.mlp.experts.58.up_proj", "model.layers.76.mlp.experts.59.up_proj", "model.layers.76.mlp.experts.60.up_proj", "model.layers.76.mlp.experts.61.up_proj", "model.layers.76.mlp.experts.62.up_proj", "model.layers.76.mlp.experts.63.up_proj", "model.layers.76.mlp.experts.64.up_proj", "model.layers.76.mlp.experts.65.up_proj", "model.layers.76.mlp.experts.66.up_proj", "model.layers.76.mlp.experts.67.up_proj", "model.layers.76.mlp.experts.68.up_proj", "model.layers.76.mlp.experts.69.up_proj", "model.layers.76.mlp.experts.70.up_proj", "model.layers.76.mlp.experts.71.up_proj", "model.layers.76.mlp.experts.72.up_proj", "model.layers.76.mlp.experts.73.up_proj", "model.layers.76.mlp.experts.74.up_proj", "model.layers.76.mlp.experts.75.up_proj", "model.layers.76.mlp.experts.76.up_proj", "model.layers.76.mlp.experts.77.up_proj", "model.layers.76.mlp.experts.78.up_proj", "model.layers.76.mlp.experts.79.up_proj", "model.layers.76.mlp.experts.80.up_proj", "model.layers.76.mlp.experts.81.up_proj", "model.layers.76.mlp.experts.82.up_proj", "model.layers.76.mlp.experts.83.up_proj", "model.layers.76.mlp.experts.84.up_proj", "model.layers.76.mlp.experts.85.up_proj", "model.layers.76.mlp.experts.86.up_proj", "model.layers.76.mlp.experts.87.up_proj", "model.layers.76.mlp.experts.88.up_proj", "model.layers.76.mlp.experts.89.up_proj", "model.layers.76.mlp.experts.90.up_proj", "model.layers.76.mlp.experts.91.up_proj", "model.layers.76.mlp.experts.92.up_proj", "model.layers.76.mlp.experts.93.up_proj", "model.layers.76.mlp.experts.94.up_proj", "model.layers.76.mlp.experts.95.up_proj", "model.layers.76.mlp.experts.96.up_proj", "model.layers.76.mlp.experts.97.up_proj", "model.layers.76.mlp.experts.98.up_proj", "model.layers.76.mlp.experts.99.up_proj", "model.layers.76.mlp.experts.100.up_proj", "model.layers.76.mlp.experts.101.up_proj", "model.layers.76.mlp.experts.102.up_proj", "model.layers.76.mlp.experts.103.up_proj", "model.layers.76.mlp.experts.104.up_proj", "model.layers.76.mlp.experts.105.up_proj", "model.layers.76.mlp.experts.106.up_proj", "model.layers.76.mlp.experts.107.up_proj", "model.layers.76.mlp.experts.108.up_proj", "model.layers.76.mlp.experts.109.up_proj", "model.layers.76.mlp.experts.110.up_proj", "model.layers.76.mlp.experts.111.up_proj", "model.layers.76.mlp.experts.112.up_proj", "model.layers.76.mlp.experts.113.up_proj", "model.layers.76.mlp.experts.114.up_proj", "model.layers.76.mlp.experts.115.up_proj", "model.layers.76.mlp.experts.116.up_proj", "model.layers.76.mlp.experts.117.up_proj", "model.layers.76.mlp.experts.118.up_proj", "model.layers.76.mlp.experts.119.up_proj", "model.layers.76.mlp.experts.120.up_proj", "model.layers.76.mlp.experts.121.up_proj", "model.layers.76.mlp.experts.122.up_proj", "model.layers.76.mlp.experts.123.up_proj", "model.layers.76.mlp.experts.124.up_proj", "model.layers.76.mlp.experts.125.up_proj", "model.layers.76.mlp.experts.126.up_proj", "model.layers.76.mlp.experts.127.up_proj", "model.layers.76.mlp.experts.128.up_proj", "model.layers.76.mlp.experts.129.up_proj", "model.layers.76.mlp.experts.130.up_proj", "model.layers.76.mlp.experts.131.up_proj", "model.layers.76.mlp.experts.132.up_proj", "model.layers.76.mlp.experts.133.up_proj", "model.layers.76.mlp.experts.134.up_proj", "model.layers.76.mlp.experts.135.up_proj", "model.layers.76.mlp.experts.136.up_proj", "model.layers.76.mlp.experts.137.up_proj", "model.layers.76.mlp.experts.138.up_proj", "model.layers.76.mlp.experts.139.up_proj", "model.layers.76.mlp.experts.140.up_proj", "model.layers.76.mlp.experts.141.up_proj", "model.layers.76.mlp.experts.142.up_proj", "model.layers.76.mlp.experts.143.up_proj", "model.layers.76.mlp.experts.144.up_proj", "model.layers.76.mlp.experts.145.up_proj", "model.layers.76.mlp.experts.146.up_proj", "model.layers.76.mlp.experts.147.up_proj", "model.layers.76.mlp.experts.148.up_proj", "model.layers.76.mlp.experts.149.up_proj", "model.layers.76.mlp.experts.150.up_proj", "model.layers.76.mlp.experts.151.up_proj", "model.layers.76.mlp.experts.152.up_proj", "model.layers.76.mlp.experts.153.up_proj", "model.layers.76.mlp.experts.154.up_proj", "model.layers.76.mlp.experts.155.up_proj", "model.layers.76.mlp.experts.156.up_proj", "model.layers.76.mlp.experts.157.up_proj", "model.layers.76.mlp.experts.158.up_proj", "model.layers.76.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.6745587345212264e-06, "dbits": 5033164800 } ] }, { "idx": 458, "layers": [ "model.layers.76.mlp.experts.0.down_proj", "model.layers.76.mlp.experts.1.down_proj", "model.layers.76.mlp.experts.2.down_proj", "model.layers.76.mlp.experts.3.down_proj", "model.layers.76.mlp.experts.4.down_proj", "model.layers.76.mlp.experts.5.down_proj", "model.layers.76.mlp.experts.6.down_proj", "model.layers.76.mlp.experts.7.down_proj", "model.layers.76.mlp.experts.8.down_proj", "model.layers.76.mlp.experts.9.down_proj", "model.layers.76.mlp.experts.10.down_proj", "model.layers.76.mlp.experts.11.down_proj", "model.layers.76.mlp.experts.12.down_proj", "model.layers.76.mlp.experts.13.down_proj", "model.layers.76.mlp.experts.14.down_proj", "model.layers.76.mlp.experts.15.down_proj", "model.layers.76.mlp.experts.16.down_proj", "model.layers.76.mlp.experts.17.down_proj", "model.layers.76.mlp.experts.18.down_proj", "model.layers.76.mlp.experts.19.down_proj", "model.layers.76.mlp.experts.20.down_proj", "model.layers.76.mlp.experts.21.down_proj", "model.layers.76.mlp.experts.22.down_proj", "model.layers.76.mlp.experts.23.down_proj", "model.layers.76.mlp.experts.24.down_proj", "model.layers.76.mlp.experts.25.down_proj", "model.layers.76.mlp.experts.26.down_proj", "model.layers.76.mlp.experts.27.down_proj", "model.layers.76.mlp.experts.28.down_proj", "model.layers.76.mlp.experts.29.down_proj", "model.layers.76.mlp.experts.30.down_proj", "model.layers.76.mlp.experts.31.down_proj", "model.layers.76.mlp.experts.32.down_proj", "model.layers.76.mlp.experts.33.down_proj", "model.layers.76.mlp.experts.34.down_proj", "model.layers.76.mlp.experts.35.down_proj", "model.layers.76.mlp.experts.36.down_proj", "model.layers.76.mlp.experts.37.down_proj", "model.layers.76.mlp.experts.38.down_proj", "model.layers.76.mlp.experts.39.down_proj", "model.layers.76.mlp.experts.40.down_proj", "model.layers.76.mlp.experts.41.down_proj", "model.layers.76.mlp.experts.42.down_proj", "model.layers.76.mlp.experts.43.down_proj", "model.layers.76.mlp.experts.44.down_proj", "model.layers.76.mlp.experts.45.down_proj", "model.layers.76.mlp.experts.46.down_proj", "model.layers.76.mlp.experts.47.down_proj", "model.layers.76.mlp.experts.48.down_proj", "model.layers.76.mlp.experts.49.down_proj", "model.layers.76.mlp.experts.50.down_proj", "model.layers.76.mlp.experts.51.down_proj", "model.layers.76.mlp.experts.52.down_proj", "model.layers.76.mlp.experts.53.down_proj", "model.layers.76.mlp.experts.54.down_proj", "model.layers.76.mlp.experts.55.down_proj", "model.layers.76.mlp.experts.56.down_proj", "model.layers.76.mlp.experts.57.down_proj", "model.layers.76.mlp.experts.58.down_proj", "model.layers.76.mlp.experts.59.down_proj", "model.layers.76.mlp.experts.60.down_proj", "model.layers.76.mlp.experts.61.down_proj", "model.layers.76.mlp.experts.62.down_proj", "model.layers.76.mlp.experts.63.down_proj", "model.layers.76.mlp.experts.64.down_proj", "model.layers.76.mlp.experts.65.down_proj", "model.layers.76.mlp.experts.66.down_proj", "model.layers.76.mlp.experts.67.down_proj", "model.layers.76.mlp.experts.68.down_proj", "model.layers.76.mlp.experts.69.down_proj", "model.layers.76.mlp.experts.70.down_proj", "model.layers.76.mlp.experts.71.down_proj", "model.layers.76.mlp.experts.72.down_proj", "model.layers.76.mlp.experts.73.down_proj", "model.layers.76.mlp.experts.74.down_proj", "model.layers.76.mlp.experts.75.down_proj", "model.layers.76.mlp.experts.76.down_proj", "model.layers.76.mlp.experts.77.down_proj", "model.layers.76.mlp.experts.78.down_proj", "model.layers.76.mlp.experts.79.down_proj", "model.layers.76.mlp.experts.80.down_proj", "model.layers.76.mlp.experts.81.down_proj", "model.layers.76.mlp.experts.82.down_proj", "model.layers.76.mlp.experts.83.down_proj", "model.layers.76.mlp.experts.84.down_proj", "model.layers.76.mlp.experts.85.down_proj", "model.layers.76.mlp.experts.86.down_proj", "model.layers.76.mlp.experts.87.down_proj", "model.layers.76.mlp.experts.88.down_proj", "model.layers.76.mlp.experts.89.down_proj", "model.layers.76.mlp.experts.90.down_proj", "model.layers.76.mlp.experts.91.down_proj", "model.layers.76.mlp.experts.92.down_proj", "model.layers.76.mlp.experts.93.down_proj", "model.layers.76.mlp.experts.94.down_proj", "model.layers.76.mlp.experts.95.down_proj", "model.layers.76.mlp.experts.96.down_proj", "model.layers.76.mlp.experts.97.down_proj", "model.layers.76.mlp.experts.98.down_proj", "model.layers.76.mlp.experts.99.down_proj", "model.layers.76.mlp.experts.100.down_proj", "model.layers.76.mlp.experts.101.down_proj", "model.layers.76.mlp.experts.102.down_proj", "model.layers.76.mlp.experts.103.down_proj", "model.layers.76.mlp.experts.104.down_proj", "model.layers.76.mlp.experts.105.down_proj", "model.layers.76.mlp.experts.106.down_proj", "model.layers.76.mlp.experts.107.down_proj", "model.layers.76.mlp.experts.108.down_proj", "model.layers.76.mlp.experts.109.down_proj", "model.layers.76.mlp.experts.110.down_proj", "model.layers.76.mlp.experts.111.down_proj", "model.layers.76.mlp.experts.112.down_proj", "model.layers.76.mlp.experts.113.down_proj", "model.layers.76.mlp.experts.114.down_proj", "model.layers.76.mlp.experts.115.down_proj", "model.layers.76.mlp.experts.116.down_proj", "model.layers.76.mlp.experts.117.down_proj", "model.layers.76.mlp.experts.118.down_proj", "model.layers.76.mlp.experts.119.down_proj", "model.layers.76.mlp.experts.120.down_proj", "model.layers.76.mlp.experts.121.down_proj", "model.layers.76.mlp.experts.122.down_proj", "model.layers.76.mlp.experts.123.down_proj", "model.layers.76.mlp.experts.124.down_proj", "model.layers.76.mlp.experts.125.down_proj", "model.layers.76.mlp.experts.126.down_proj", "model.layers.76.mlp.experts.127.down_proj", "model.layers.76.mlp.experts.128.down_proj", "model.layers.76.mlp.experts.129.down_proj", "model.layers.76.mlp.experts.130.down_proj", "model.layers.76.mlp.experts.131.down_proj", "model.layers.76.mlp.experts.132.down_proj", "model.layers.76.mlp.experts.133.down_proj", "model.layers.76.mlp.experts.134.down_proj", "model.layers.76.mlp.experts.135.down_proj", "model.layers.76.mlp.experts.136.down_proj", "model.layers.76.mlp.experts.137.down_proj", "model.layers.76.mlp.experts.138.down_proj", "model.layers.76.mlp.experts.139.down_proj", "model.layers.76.mlp.experts.140.down_proj", "model.layers.76.mlp.experts.141.down_proj", "model.layers.76.mlp.experts.142.down_proj", "model.layers.76.mlp.experts.143.down_proj", "model.layers.76.mlp.experts.144.down_proj", "model.layers.76.mlp.experts.145.down_proj", "model.layers.76.mlp.experts.146.down_proj", "model.layers.76.mlp.experts.147.down_proj", "model.layers.76.mlp.experts.148.down_proj", "model.layers.76.mlp.experts.149.down_proj", "model.layers.76.mlp.experts.150.down_proj", "model.layers.76.mlp.experts.151.down_proj", "model.layers.76.mlp.experts.152.down_proj", "model.layers.76.mlp.experts.153.down_proj", "model.layers.76.mlp.experts.154.down_proj", "model.layers.76.mlp.experts.155.down_proj", "model.layers.76.mlp.experts.156.down_proj", "model.layers.76.mlp.experts.157.down_proj", "model.layers.76.mlp.experts.158.down_proj", "model.layers.76.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.023499109782235e-06, "dbits": 2516582400 } ] }, { "idx": 459, "layers": [ "model.layers.77.self_attn.q_proj" ], "candidates": [ { "dkld": 3.667926648631258e-07, "dbits": 125829120 } ] }, { "idx": 460, "layers": [ "model.layers.77.self_attn.k_proj", "model.layers.77.self_attn.v_proj" ], "candidates": [ { "dkld": -6.22987863607683e-07, "dbits": 20971520 } ] }, { "idx": 461, "layers": [ "model.layers.77.self_attn.o_proj" ], "candidates": [ { "dkld": 1.2857373803854422e-06, "dbits": 125829120 } ] }, { "idx": 462, "layers": [ "model.layers.77.mlp.shared_experts.gate_proj", "model.layers.77.mlp.shared_experts.up_proj", "model.layers.77.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.7953734388575175e-06, "dbits": 47185920 } ] }, { "idx": 463, "layers": [ "model.layers.77.mlp.experts.0.gate_proj", "model.layers.77.mlp.experts.1.gate_proj", "model.layers.77.mlp.experts.2.gate_proj", "model.layers.77.mlp.experts.3.gate_proj", "model.layers.77.mlp.experts.4.gate_proj", "model.layers.77.mlp.experts.5.gate_proj", "model.layers.77.mlp.experts.6.gate_proj", "model.layers.77.mlp.experts.7.gate_proj", "model.layers.77.mlp.experts.8.gate_proj", "model.layers.77.mlp.experts.9.gate_proj", "model.layers.77.mlp.experts.10.gate_proj", "model.layers.77.mlp.experts.11.gate_proj", "model.layers.77.mlp.experts.12.gate_proj", "model.layers.77.mlp.experts.13.gate_proj", "model.layers.77.mlp.experts.14.gate_proj", "model.layers.77.mlp.experts.15.gate_proj", "model.layers.77.mlp.experts.16.gate_proj", "model.layers.77.mlp.experts.17.gate_proj", "model.layers.77.mlp.experts.18.gate_proj", "model.layers.77.mlp.experts.19.gate_proj", "model.layers.77.mlp.experts.20.gate_proj", "model.layers.77.mlp.experts.21.gate_proj", "model.layers.77.mlp.experts.22.gate_proj", "model.layers.77.mlp.experts.23.gate_proj", "model.layers.77.mlp.experts.24.gate_proj", "model.layers.77.mlp.experts.25.gate_proj", "model.layers.77.mlp.experts.26.gate_proj", "model.layers.77.mlp.experts.27.gate_proj", "model.layers.77.mlp.experts.28.gate_proj", "model.layers.77.mlp.experts.29.gate_proj", "model.layers.77.mlp.experts.30.gate_proj", "model.layers.77.mlp.experts.31.gate_proj", "model.layers.77.mlp.experts.32.gate_proj", "model.layers.77.mlp.experts.33.gate_proj", "model.layers.77.mlp.experts.34.gate_proj", "model.layers.77.mlp.experts.35.gate_proj", "model.layers.77.mlp.experts.36.gate_proj", "model.layers.77.mlp.experts.37.gate_proj", "model.layers.77.mlp.experts.38.gate_proj", "model.layers.77.mlp.experts.39.gate_proj", "model.layers.77.mlp.experts.40.gate_proj", "model.layers.77.mlp.experts.41.gate_proj", "model.layers.77.mlp.experts.42.gate_proj", "model.layers.77.mlp.experts.43.gate_proj", "model.layers.77.mlp.experts.44.gate_proj", "model.layers.77.mlp.experts.45.gate_proj", "model.layers.77.mlp.experts.46.gate_proj", "model.layers.77.mlp.experts.47.gate_proj", "model.layers.77.mlp.experts.48.gate_proj", "model.layers.77.mlp.experts.49.gate_proj", "model.layers.77.mlp.experts.50.gate_proj", "model.layers.77.mlp.experts.51.gate_proj", "model.layers.77.mlp.experts.52.gate_proj", "model.layers.77.mlp.experts.53.gate_proj", "model.layers.77.mlp.experts.54.gate_proj", "model.layers.77.mlp.experts.55.gate_proj", "model.layers.77.mlp.experts.56.gate_proj", "model.layers.77.mlp.experts.57.gate_proj", "model.layers.77.mlp.experts.58.gate_proj", "model.layers.77.mlp.experts.59.gate_proj", "model.layers.77.mlp.experts.60.gate_proj", "model.layers.77.mlp.experts.61.gate_proj", "model.layers.77.mlp.experts.62.gate_proj", "model.layers.77.mlp.experts.63.gate_proj", "model.layers.77.mlp.experts.64.gate_proj", "model.layers.77.mlp.experts.65.gate_proj", "model.layers.77.mlp.experts.66.gate_proj", "model.layers.77.mlp.experts.67.gate_proj", "model.layers.77.mlp.experts.68.gate_proj", "model.layers.77.mlp.experts.69.gate_proj", "model.layers.77.mlp.experts.70.gate_proj", "model.layers.77.mlp.experts.71.gate_proj", "model.layers.77.mlp.experts.72.gate_proj", "model.layers.77.mlp.experts.73.gate_proj", "model.layers.77.mlp.experts.74.gate_proj", "model.layers.77.mlp.experts.75.gate_proj", "model.layers.77.mlp.experts.76.gate_proj", "model.layers.77.mlp.experts.77.gate_proj", "model.layers.77.mlp.experts.78.gate_proj", "model.layers.77.mlp.experts.79.gate_proj", "model.layers.77.mlp.experts.80.gate_proj", "model.layers.77.mlp.experts.81.gate_proj", "model.layers.77.mlp.experts.82.gate_proj", "model.layers.77.mlp.experts.83.gate_proj", "model.layers.77.mlp.experts.84.gate_proj", "model.layers.77.mlp.experts.85.gate_proj", "model.layers.77.mlp.experts.86.gate_proj", "model.layers.77.mlp.experts.87.gate_proj", "model.layers.77.mlp.experts.88.gate_proj", "model.layers.77.mlp.experts.89.gate_proj", "model.layers.77.mlp.experts.90.gate_proj", "model.layers.77.mlp.experts.91.gate_proj", "model.layers.77.mlp.experts.92.gate_proj", "model.layers.77.mlp.experts.93.gate_proj", "model.layers.77.mlp.experts.94.gate_proj", "model.layers.77.mlp.experts.95.gate_proj", "model.layers.77.mlp.experts.96.gate_proj", "model.layers.77.mlp.experts.97.gate_proj", "model.layers.77.mlp.experts.98.gate_proj", "model.layers.77.mlp.experts.99.gate_proj", "model.layers.77.mlp.experts.100.gate_proj", "model.layers.77.mlp.experts.101.gate_proj", "model.layers.77.mlp.experts.102.gate_proj", "model.layers.77.mlp.experts.103.gate_proj", "model.layers.77.mlp.experts.104.gate_proj", "model.layers.77.mlp.experts.105.gate_proj", "model.layers.77.mlp.experts.106.gate_proj", "model.layers.77.mlp.experts.107.gate_proj", "model.layers.77.mlp.experts.108.gate_proj", "model.layers.77.mlp.experts.109.gate_proj", "model.layers.77.mlp.experts.110.gate_proj", "model.layers.77.mlp.experts.111.gate_proj", "model.layers.77.mlp.experts.112.gate_proj", "model.layers.77.mlp.experts.113.gate_proj", "model.layers.77.mlp.experts.114.gate_proj", "model.layers.77.mlp.experts.115.gate_proj", "model.layers.77.mlp.experts.116.gate_proj", "model.layers.77.mlp.experts.117.gate_proj", "model.layers.77.mlp.experts.118.gate_proj", "model.layers.77.mlp.experts.119.gate_proj", "model.layers.77.mlp.experts.120.gate_proj", "model.layers.77.mlp.experts.121.gate_proj", "model.layers.77.mlp.experts.122.gate_proj", "model.layers.77.mlp.experts.123.gate_proj", "model.layers.77.mlp.experts.124.gate_proj", "model.layers.77.mlp.experts.125.gate_proj", "model.layers.77.mlp.experts.126.gate_proj", "model.layers.77.mlp.experts.127.gate_proj", "model.layers.77.mlp.experts.128.gate_proj", "model.layers.77.mlp.experts.129.gate_proj", "model.layers.77.mlp.experts.130.gate_proj", "model.layers.77.mlp.experts.131.gate_proj", "model.layers.77.mlp.experts.132.gate_proj", "model.layers.77.mlp.experts.133.gate_proj", "model.layers.77.mlp.experts.134.gate_proj", "model.layers.77.mlp.experts.135.gate_proj", "model.layers.77.mlp.experts.136.gate_proj", "model.layers.77.mlp.experts.137.gate_proj", "model.layers.77.mlp.experts.138.gate_proj", "model.layers.77.mlp.experts.139.gate_proj", "model.layers.77.mlp.experts.140.gate_proj", "model.layers.77.mlp.experts.141.gate_proj", "model.layers.77.mlp.experts.142.gate_proj", "model.layers.77.mlp.experts.143.gate_proj", "model.layers.77.mlp.experts.144.gate_proj", "model.layers.77.mlp.experts.145.gate_proj", "model.layers.77.mlp.experts.146.gate_proj", "model.layers.77.mlp.experts.147.gate_proj", "model.layers.77.mlp.experts.148.gate_proj", "model.layers.77.mlp.experts.149.gate_proj", "model.layers.77.mlp.experts.150.gate_proj", "model.layers.77.mlp.experts.151.gate_proj", "model.layers.77.mlp.experts.152.gate_proj", "model.layers.77.mlp.experts.153.gate_proj", "model.layers.77.mlp.experts.154.gate_proj", "model.layers.77.mlp.experts.155.gate_proj", "model.layers.77.mlp.experts.156.gate_proj", "model.layers.77.mlp.experts.157.gate_proj", "model.layers.77.mlp.experts.158.gate_proj", "model.layers.77.mlp.experts.159.gate_proj", "model.layers.77.mlp.experts.0.up_proj", "model.layers.77.mlp.experts.1.up_proj", "model.layers.77.mlp.experts.2.up_proj", "model.layers.77.mlp.experts.3.up_proj", "model.layers.77.mlp.experts.4.up_proj", "model.layers.77.mlp.experts.5.up_proj", "model.layers.77.mlp.experts.6.up_proj", "model.layers.77.mlp.experts.7.up_proj", "model.layers.77.mlp.experts.8.up_proj", "model.layers.77.mlp.experts.9.up_proj", "model.layers.77.mlp.experts.10.up_proj", "model.layers.77.mlp.experts.11.up_proj", "model.layers.77.mlp.experts.12.up_proj", "model.layers.77.mlp.experts.13.up_proj", "model.layers.77.mlp.experts.14.up_proj", "model.layers.77.mlp.experts.15.up_proj", "model.layers.77.mlp.experts.16.up_proj", "model.layers.77.mlp.experts.17.up_proj", "model.layers.77.mlp.experts.18.up_proj", "model.layers.77.mlp.experts.19.up_proj", "model.layers.77.mlp.experts.20.up_proj", "model.layers.77.mlp.experts.21.up_proj", "model.layers.77.mlp.experts.22.up_proj", "model.layers.77.mlp.experts.23.up_proj", "model.layers.77.mlp.experts.24.up_proj", "model.layers.77.mlp.experts.25.up_proj", "model.layers.77.mlp.experts.26.up_proj", "model.layers.77.mlp.experts.27.up_proj", "model.layers.77.mlp.experts.28.up_proj", "model.layers.77.mlp.experts.29.up_proj", "model.layers.77.mlp.experts.30.up_proj", "model.layers.77.mlp.experts.31.up_proj", "model.layers.77.mlp.experts.32.up_proj", "model.layers.77.mlp.experts.33.up_proj", "model.layers.77.mlp.experts.34.up_proj", "model.layers.77.mlp.experts.35.up_proj", "model.layers.77.mlp.experts.36.up_proj", "model.layers.77.mlp.experts.37.up_proj", "model.layers.77.mlp.experts.38.up_proj", "model.layers.77.mlp.experts.39.up_proj", "model.layers.77.mlp.experts.40.up_proj", "model.layers.77.mlp.experts.41.up_proj", "model.layers.77.mlp.experts.42.up_proj", "model.layers.77.mlp.experts.43.up_proj", "model.layers.77.mlp.experts.44.up_proj", "model.layers.77.mlp.experts.45.up_proj", "model.layers.77.mlp.experts.46.up_proj", "model.layers.77.mlp.experts.47.up_proj", "model.layers.77.mlp.experts.48.up_proj", "model.layers.77.mlp.experts.49.up_proj", "model.layers.77.mlp.experts.50.up_proj", "model.layers.77.mlp.experts.51.up_proj", "model.layers.77.mlp.experts.52.up_proj", "model.layers.77.mlp.experts.53.up_proj", "model.layers.77.mlp.experts.54.up_proj", "model.layers.77.mlp.experts.55.up_proj", "model.layers.77.mlp.experts.56.up_proj", "model.layers.77.mlp.experts.57.up_proj", "model.layers.77.mlp.experts.58.up_proj", "model.layers.77.mlp.experts.59.up_proj", "model.layers.77.mlp.experts.60.up_proj", "model.layers.77.mlp.experts.61.up_proj", "model.layers.77.mlp.experts.62.up_proj", "model.layers.77.mlp.experts.63.up_proj", "model.layers.77.mlp.experts.64.up_proj", "model.layers.77.mlp.experts.65.up_proj", "model.layers.77.mlp.experts.66.up_proj", "model.layers.77.mlp.experts.67.up_proj", "model.layers.77.mlp.experts.68.up_proj", "model.layers.77.mlp.experts.69.up_proj", "model.layers.77.mlp.experts.70.up_proj", "model.layers.77.mlp.experts.71.up_proj", "model.layers.77.mlp.experts.72.up_proj", "model.layers.77.mlp.experts.73.up_proj", "model.layers.77.mlp.experts.74.up_proj", "model.layers.77.mlp.experts.75.up_proj", "model.layers.77.mlp.experts.76.up_proj", "model.layers.77.mlp.experts.77.up_proj", "model.layers.77.mlp.experts.78.up_proj", "model.layers.77.mlp.experts.79.up_proj", "model.layers.77.mlp.experts.80.up_proj", "model.layers.77.mlp.experts.81.up_proj", "model.layers.77.mlp.experts.82.up_proj", "model.layers.77.mlp.experts.83.up_proj", "model.layers.77.mlp.experts.84.up_proj", "model.layers.77.mlp.experts.85.up_proj", "model.layers.77.mlp.experts.86.up_proj", "model.layers.77.mlp.experts.87.up_proj", "model.layers.77.mlp.experts.88.up_proj", "model.layers.77.mlp.experts.89.up_proj", "model.layers.77.mlp.experts.90.up_proj", "model.layers.77.mlp.experts.91.up_proj", "model.layers.77.mlp.experts.92.up_proj", "model.layers.77.mlp.experts.93.up_proj", "model.layers.77.mlp.experts.94.up_proj", "model.layers.77.mlp.experts.95.up_proj", "model.layers.77.mlp.experts.96.up_proj", "model.layers.77.mlp.experts.97.up_proj", "model.layers.77.mlp.experts.98.up_proj", "model.layers.77.mlp.experts.99.up_proj", "model.layers.77.mlp.experts.100.up_proj", "model.layers.77.mlp.experts.101.up_proj", "model.layers.77.mlp.experts.102.up_proj", "model.layers.77.mlp.experts.103.up_proj", "model.layers.77.mlp.experts.104.up_proj", "model.layers.77.mlp.experts.105.up_proj", "model.layers.77.mlp.experts.106.up_proj", "model.layers.77.mlp.experts.107.up_proj", "model.layers.77.mlp.experts.108.up_proj", "model.layers.77.mlp.experts.109.up_proj", "model.layers.77.mlp.experts.110.up_proj", "model.layers.77.mlp.experts.111.up_proj", "model.layers.77.mlp.experts.112.up_proj", "model.layers.77.mlp.experts.113.up_proj", "model.layers.77.mlp.experts.114.up_proj", "model.layers.77.mlp.experts.115.up_proj", "model.layers.77.mlp.experts.116.up_proj", "model.layers.77.mlp.experts.117.up_proj", "model.layers.77.mlp.experts.118.up_proj", "model.layers.77.mlp.experts.119.up_proj", "model.layers.77.mlp.experts.120.up_proj", "model.layers.77.mlp.experts.121.up_proj", "model.layers.77.mlp.experts.122.up_proj", "model.layers.77.mlp.experts.123.up_proj", "model.layers.77.mlp.experts.124.up_proj", "model.layers.77.mlp.experts.125.up_proj", "model.layers.77.mlp.experts.126.up_proj", "model.layers.77.mlp.experts.127.up_proj", "model.layers.77.mlp.experts.128.up_proj", "model.layers.77.mlp.experts.129.up_proj", "model.layers.77.mlp.experts.130.up_proj", "model.layers.77.mlp.experts.131.up_proj", "model.layers.77.mlp.experts.132.up_proj", "model.layers.77.mlp.experts.133.up_proj", "model.layers.77.mlp.experts.134.up_proj", "model.layers.77.mlp.experts.135.up_proj", "model.layers.77.mlp.experts.136.up_proj", "model.layers.77.mlp.experts.137.up_proj", "model.layers.77.mlp.experts.138.up_proj", "model.layers.77.mlp.experts.139.up_proj", "model.layers.77.mlp.experts.140.up_proj", "model.layers.77.mlp.experts.141.up_proj", "model.layers.77.mlp.experts.142.up_proj", "model.layers.77.mlp.experts.143.up_proj", "model.layers.77.mlp.experts.144.up_proj", "model.layers.77.mlp.experts.145.up_proj", "model.layers.77.mlp.experts.146.up_proj", "model.layers.77.mlp.experts.147.up_proj", "model.layers.77.mlp.experts.148.up_proj", "model.layers.77.mlp.experts.149.up_proj", "model.layers.77.mlp.experts.150.up_proj", "model.layers.77.mlp.experts.151.up_proj", "model.layers.77.mlp.experts.152.up_proj", "model.layers.77.mlp.experts.153.up_proj", "model.layers.77.mlp.experts.154.up_proj", "model.layers.77.mlp.experts.155.up_proj", "model.layers.77.mlp.experts.156.up_proj", "model.layers.77.mlp.experts.157.up_proj", "model.layers.77.mlp.experts.158.up_proj", "model.layers.77.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.198869181797342e-07, "dbits": 5033164800 } ] }, { "idx": 464, "layers": [ "model.layers.77.mlp.experts.0.down_proj", "model.layers.77.mlp.experts.1.down_proj", "model.layers.77.mlp.experts.2.down_proj", "model.layers.77.mlp.experts.3.down_proj", "model.layers.77.mlp.experts.4.down_proj", "model.layers.77.mlp.experts.5.down_proj", "model.layers.77.mlp.experts.6.down_proj", "model.layers.77.mlp.experts.7.down_proj", "model.layers.77.mlp.experts.8.down_proj", "model.layers.77.mlp.experts.9.down_proj", "model.layers.77.mlp.experts.10.down_proj", "model.layers.77.mlp.experts.11.down_proj", "model.layers.77.mlp.experts.12.down_proj", "model.layers.77.mlp.experts.13.down_proj", "model.layers.77.mlp.experts.14.down_proj", "model.layers.77.mlp.experts.15.down_proj", "model.layers.77.mlp.experts.16.down_proj", "model.layers.77.mlp.experts.17.down_proj", "model.layers.77.mlp.experts.18.down_proj", "model.layers.77.mlp.experts.19.down_proj", "model.layers.77.mlp.experts.20.down_proj", "model.layers.77.mlp.experts.21.down_proj", "model.layers.77.mlp.experts.22.down_proj", "model.layers.77.mlp.experts.23.down_proj", "model.layers.77.mlp.experts.24.down_proj", "model.layers.77.mlp.experts.25.down_proj", "model.layers.77.mlp.experts.26.down_proj", "model.layers.77.mlp.experts.27.down_proj", "model.layers.77.mlp.experts.28.down_proj", "model.layers.77.mlp.experts.29.down_proj", "model.layers.77.mlp.experts.30.down_proj", "model.layers.77.mlp.experts.31.down_proj", "model.layers.77.mlp.experts.32.down_proj", "model.layers.77.mlp.experts.33.down_proj", "model.layers.77.mlp.experts.34.down_proj", "model.layers.77.mlp.experts.35.down_proj", "model.layers.77.mlp.experts.36.down_proj", "model.layers.77.mlp.experts.37.down_proj", "model.layers.77.mlp.experts.38.down_proj", "model.layers.77.mlp.experts.39.down_proj", "model.layers.77.mlp.experts.40.down_proj", "model.layers.77.mlp.experts.41.down_proj", "model.layers.77.mlp.experts.42.down_proj", "model.layers.77.mlp.experts.43.down_proj", "model.layers.77.mlp.experts.44.down_proj", "model.layers.77.mlp.experts.45.down_proj", "model.layers.77.mlp.experts.46.down_proj", "model.layers.77.mlp.experts.47.down_proj", "model.layers.77.mlp.experts.48.down_proj", "model.layers.77.mlp.experts.49.down_proj", "model.layers.77.mlp.experts.50.down_proj", "model.layers.77.mlp.experts.51.down_proj", "model.layers.77.mlp.experts.52.down_proj", "model.layers.77.mlp.experts.53.down_proj", "model.layers.77.mlp.experts.54.down_proj", "model.layers.77.mlp.experts.55.down_proj", "model.layers.77.mlp.experts.56.down_proj", "model.layers.77.mlp.experts.57.down_proj", "model.layers.77.mlp.experts.58.down_proj", "model.layers.77.mlp.experts.59.down_proj", "model.layers.77.mlp.experts.60.down_proj", "model.layers.77.mlp.experts.61.down_proj", "model.layers.77.mlp.experts.62.down_proj", "model.layers.77.mlp.experts.63.down_proj", "model.layers.77.mlp.experts.64.down_proj", "model.layers.77.mlp.experts.65.down_proj", "model.layers.77.mlp.experts.66.down_proj", "model.layers.77.mlp.experts.67.down_proj", "model.layers.77.mlp.experts.68.down_proj", "model.layers.77.mlp.experts.69.down_proj", "model.layers.77.mlp.experts.70.down_proj", "model.layers.77.mlp.experts.71.down_proj", "model.layers.77.mlp.experts.72.down_proj", "model.layers.77.mlp.experts.73.down_proj", "model.layers.77.mlp.experts.74.down_proj", "model.layers.77.mlp.experts.75.down_proj", "model.layers.77.mlp.experts.76.down_proj", "model.layers.77.mlp.experts.77.down_proj", "model.layers.77.mlp.experts.78.down_proj", "model.layers.77.mlp.experts.79.down_proj", "model.layers.77.mlp.experts.80.down_proj", "model.layers.77.mlp.experts.81.down_proj", "model.layers.77.mlp.experts.82.down_proj", "model.layers.77.mlp.experts.83.down_proj", "model.layers.77.mlp.experts.84.down_proj", "model.layers.77.mlp.experts.85.down_proj", "model.layers.77.mlp.experts.86.down_proj", "model.layers.77.mlp.experts.87.down_proj", "model.layers.77.mlp.experts.88.down_proj", "model.layers.77.mlp.experts.89.down_proj", "model.layers.77.mlp.experts.90.down_proj", "model.layers.77.mlp.experts.91.down_proj", "model.layers.77.mlp.experts.92.down_proj", "model.layers.77.mlp.experts.93.down_proj", "model.layers.77.mlp.experts.94.down_proj", "model.layers.77.mlp.experts.95.down_proj", "model.layers.77.mlp.experts.96.down_proj", "model.layers.77.mlp.experts.97.down_proj", "model.layers.77.mlp.experts.98.down_proj", "model.layers.77.mlp.experts.99.down_proj", "model.layers.77.mlp.experts.100.down_proj", "model.layers.77.mlp.experts.101.down_proj", "model.layers.77.mlp.experts.102.down_proj", "model.layers.77.mlp.experts.103.down_proj", "model.layers.77.mlp.experts.104.down_proj", "model.layers.77.mlp.experts.105.down_proj", "model.layers.77.mlp.experts.106.down_proj", "model.layers.77.mlp.experts.107.down_proj", "model.layers.77.mlp.experts.108.down_proj", "model.layers.77.mlp.experts.109.down_proj", "model.layers.77.mlp.experts.110.down_proj", "model.layers.77.mlp.experts.111.down_proj", "model.layers.77.mlp.experts.112.down_proj", "model.layers.77.mlp.experts.113.down_proj", "model.layers.77.mlp.experts.114.down_proj", "model.layers.77.mlp.experts.115.down_proj", "model.layers.77.mlp.experts.116.down_proj", "model.layers.77.mlp.experts.117.down_proj", "model.layers.77.mlp.experts.118.down_proj", "model.layers.77.mlp.experts.119.down_proj", "model.layers.77.mlp.experts.120.down_proj", "model.layers.77.mlp.experts.121.down_proj", "model.layers.77.mlp.experts.122.down_proj", "model.layers.77.mlp.experts.123.down_proj", "model.layers.77.mlp.experts.124.down_proj", "model.layers.77.mlp.experts.125.down_proj", "model.layers.77.mlp.experts.126.down_proj", "model.layers.77.mlp.experts.127.down_proj", "model.layers.77.mlp.experts.128.down_proj", "model.layers.77.mlp.experts.129.down_proj", "model.layers.77.mlp.experts.130.down_proj", "model.layers.77.mlp.experts.131.down_proj", "model.layers.77.mlp.experts.132.down_proj", "model.layers.77.mlp.experts.133.down_proj", "model.layers.77.mlp.experts.134.down_proj", "model.layers.77.mlp.experts.135.down_proj", "model.layers.77.mlp.experts.136.down_proj", "model.layers.77.mlp.experts.137.down_proj", "model.layers.77.mlp.experts.138.down_proj", "model.layers.77.mlp.experts.139.down_proj", "model.layers.77.mlp.experts.140.down_proj", "model.layers.77.mlp.experts.141.down_proj", "model.layers.77.mlp.experts.142.down_proj", "model.layers.77.mlp.experts.143.down_proj", "model.layers.77.mlp.experts.144.down_proj", "model.layers.77.mlp.experts.145.down_proj", "model.layers.77.mlp.experts.146.down_proj", "model.layers.77.mlp.experts.147.down_proj", "model.layers.77.mlp.experts.148.down_proj", "model.layers.77.mlp.experts.149.down_proj", "model.layers.77.mlp.experts.150.down_proj", "model.layers.77.mlp.experts.151.down_proj", "model.layers.77.mlp.experts.152.down_proj", "model.layers.77.mlp.experts.153.down_proj", "model.layers.77.mlp.experts.154.down_proj", "model.layers.77.mlp.experts.155.down_proj", "model.layers.77.mlp.experts.156.down_proj", "model.layers.77.mlp.experts.157.down_proj", "model.layers.77.mlp.experts.158.down_proj", "model.layers.77.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.8246315196156935e-06, "dbits": 2516582400 } ] }, { "idx": 465, "layers": [ "model.layers.78.self_attn.q_proj" ], "candidates": [ { "dkld": 2.32328020501884e-06, "dbits": 125829120 } ] }, { "idx": 466, "layers": [ "model.layers.78.self_attn.k_proj", "model.layers.78.self_attn.v_proj" ], "candidates": [ { "dkld": 4.099763464182637e-06, "dbits": 20971520 } ] }, { "idx": 467, "layers": [ "model.layers.78.self_attn.o_proj" ], "candidates": [ { "dkld": 2.0101229893043854e-06, "dbits": 125829120 } ] }, { "idx": 468, "layers": [ "model.layers.78.mlp.shared_experts.gate_proj", "model.layers.78.mlp.shared_experts.up_proj", "model.layers.78.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.2627529688179927e-06, "dbits": 47185920 } ] }, { "idx": 469, "layers": [ "model.layers.78.mlp.experts.0.gate_proj", "model.layers.78.mlp.experts.1.gate_proj", "model.layers.78.mlp.experts.2.gate_proj", "model.layers.78.mlp.experts.3.gate_proj", "model.layers.78.mlp.experts.4.gate_proj", "model.layers.78.mlp.experts.5.gate_proj", "model.layers.78.mlp.experts.6.gate_proj", "model.layers.78.mlp.experts.7.gate_proj", "model.layers.78.mlp.experts.8.gate_proj", "model.layers.78.mlp.experts.9.gate_proj", "model.layers.78.mlp.experts.10.gate_proj", "model.layers.78.mlp.experts.11.gate_proj", "model.layers.78.mlp.experts.12.gate_proj", "model.layers.78.mlp.experts.13.gate_proj", "model.layers.78.mlp.experts.14.gate_proj", "model.layers.78.mlp.experts.15.gate_proj", "model.layers.78.mlp.experts.16.gate_proj", "model.layers.78.mlp.experts.17.gate_proj", "model.layers.78.mlp.experts.18.gate_proj", "model.layers.78.mlp.experts.19.gate_proj", "model.layers.78.mlp.experts.20.gate_proj", "model.layers.78.mlp.experts.21.gate_proj", "model.layers.78.mlp.experts.22.gate_proj", "model.layers.78.mlp.experts.23.gate_proj", "model.layers.78.mlp.experts.24.gate_proj", "model.layers.78.mlp.experts.25.gate_proj", "model.layers.78.mlp.experts.26.gate_proj", "model.layers.78.mlp.experts.27.gate_proj", "model.layers.78.mlp.experts.28.gate_proj", "model.layers.78.mlp.experts.29.gate_proj", "model.layers.78.mlp.experts.30.gate_proj", "model.layers.78.mlp.experts.31.gate_proj", "model.layers.78.mlp.experts.32.gate_proj", "model.layers.78.mlp.experts.33.gate_proj", "model.layers.78.mlp.experts.34.gate_proj", "model.layers.78.mlp.experts.35.gate_proj", "model.layers.78.mlp.experts.36.gate_proj", "model.layers.78.mlp.experts.37.gate_proj", "model.layers.78.mlp.experts.38.gate_proj", "model.layers.78.mlp.experts.39.gate_proj", "model.layers.78.mlp.experts.40.gate_proj", "model.layers.78.mlp.experts.41.gate_proj", "model.layers.78.mlp.experts.42.gate_proj", "model.layers.78.mlp.experts.43.gate_proj", "model.layers.78.mlp.experts.44.gate_proj", "model.layers.78.mlp.experts.45.gate_proj", "model.layers.78.mlp.experts.46.gate_proj", "model.layers.78.mlp.experts.47.gate_proj", "model.layers.78.mlp.experts.48.gate_proj", "model.layers.78.mlp.experts.49.gate_proj", "model.layers.78.mlp.experts.50.gate_proj", "model.layers.78.mlp.experts.51.gate_proj", "model.layers.78.mlp.experts.52.gate_proj", "model.layers.78.mlp.experts.53.gate_proj", "model.layers.78.mlp.experts.54.gate_proj", "model.layers.78.mlp.experts.55.gate_proj", "model.layers.78.mlp.experts.56.gate_proj", "model.layers.78.mlp.experts.57.gate_proj", "model.layers.78.mlp.experts.58.gate_proj", "model.layers.78.mlp.experts.59.gate_proj", "model.layers.78.mlp.experts.60.gate_proj", "model.layers.78.mlp.experts.61.gate_proj", "model.layers.78.mlp.experts.62.gate_proj", "model.layers.78.mlp.experts.63.gate_proj", "model.layers.78.mlp.experts.64.gate_proj", "model.layers.78.mlp.experts.65.gate_proj", "model.layers.78.mlp.experts.66.gate_proj", "model.layers.78.mlp.experts.67.gate_proj", "model.layers.78.mlp.experts.68.gate_proj", "model.layers.78.mlp.experts.69.gate_proj", "model.layers.78.mlp.experts.70.gate_proj", "model.layers.78.mlp.experts.71.gate_proj", "model.layers.78.mlp.experts.72.gate_proj", "model.layers.78.mlp.experts.73.gate_proj", "model.layers.78.mlp.experts.74.gate_proj", "model.layers.78.mlp.experts.75.gate_proj", "model.layers.78.mlp.experts.76.gate_proj", "model.layers.78.mlp.experts.77.gate_proj", "model.layers.78.mlp.experts.78.gate_proj", "model.layers.78.mlp.experts.79.gate_proj", "model.layers.78.mlp.experts.80.gate_proj", "model.layers.78.mlp.experts.81.gate_proj", "model.layers.78.mlp.experts.82.gate_proj", "model.layers.78.mlp.experts.83.gate_proj", "model.layers.78.mlp.experts.84.gate_proj", "model.layers.78.mlp.experts.85.gate_proj", "model.layers.78.mlp.experts.86.gate_proj", "model.layers.78.mlp.experts.87.gate_proj", "model.layers.78.mlp.experts.88.gate_proj", "model.layers.78.mlp.experts.89.gate_proj", "model.layers.78.mlp.experts.90.gate_proj", "model.layers.78.mlp.experts.91.gate_proj", "model.layers.78.mlp.experts.92.gate_proj", "model.layers.78.mlp.experts.93.gate_proj", "model.layers.78.mlp.experts.94.gate_proj", "model.layers.78.mlp.experts.95.gate_proj", "model.layers.78.mlp.experts.96.gate_proj", "model.layers.78.mlp.experts.97.gate_proj", "model.layers.78.mlp.experts.98.gate_proj", "model.layers.78.mlp.experts.99.gate_proj", "model.layers.78.mlp.experts.100.gate_proj", "model.layers.78.mlp.experts.101.gate_proj", "model.layers.78.mlp.experts.102.gate_proj", "model.layers.78.mlp.experts.103.gate_proj", "model.layers.78.mlp.experts.104.gate_proj", "model.layers.78.mlp.experts.105.gate_proj", "model.layers.78.mlp.experts.106.gate_proj", "model.layers.78.mlp.experts.107.gate_proj", "model.layers.78.mlp.experts.108.gate_proj", "model.layers.78.mlp.experts.109.gate_proj", "model.layers.78.mlp.experts.110.gate_proj", "model.layers.78.mlp.experts.111.gate_proj", "model.layers.78.mlp.experts.112.gate_proj", "model.layers.78.mlp.experts.113.gate_proj", "model.layers.78.mlp.experts.114.gate_proj", "model.layers.78.mlp.experts.115.gate_proj", "model.layers.78.mlp.experts.116.gate_proj", "model.layers.78.mlp.experts.117.gate_proj", "model.layers.78.mlp.experts.118.gate_proj", "model.layers.78.mlp.experts.119.gate_proj", "model.layers.78.mlp.experts.120.gate_proj", "model.layers.78.mlp.experts.121.gate_proj", "model.layers.78.mlp.experts.122.gate_proj", "model.layers.78.mlp.experts.123.gate_proj", "model.layers.78.mlp.experts.124.gate_proj", "model.layers.78.mlp.experts.125.gate_proj", "model.layers.78.mlp.experts.126.gate_proj", "model.layers.78.mlp.experts.127.gate_proj", "model.layers.78.mlp.experts.128.gate_proj", "model.layers.78.mlp.experts.129.gate_proj", "model.layers.78.mlp.experts.130.gate_proj", "model.layers.78.mlp.experts.131.gate_proj", "model.layers.78.mlp.experts.132.gate_proj", "model.layers.78.mlp.experts.133.gate_proj", "model.layers.78.mlp.experts.134.gate_proj", "model.layers.78.mlp.experts.135.gate_proj", "model.layers.78.mlp.experts.136.gate_proj", "model.layers.78.mlp.experts.137.gate_proj", "model.layers.78.mlp.experts.138.gate_proj", "model.layers.78.mlp.experts.139.gate_proj", "model.layers.78.mlp.experts.140.gate_proj", "model.layers.78.mlp.experts.141.gate_proj", "model.layers.78.mlp.experts.142.gate_proj", "model.layers.78.mlp.experts.143.gate_proj", "model.layers.78.mlp.experts.144.gate_proj", "model.layers.78.mlp.experts.145.gate_proj", "model.layers.78.mlp.experts.146.gate_proj", "model.layers.78.mlp.experts.147.gate_proj", "model.layers.78.mlp.experts.148.gate_proj", "model.layers.78.mlp.experts.149.gate_proj", "model.layers.78.mlp.experts.150.gate_proj", "model.layers.78.mlp.experts.151.gate_proj", "model.layers.78.mlp.experts.152.gate_proj", "model.layers.78.mlp.experts.153.gate_proj", "model.layers.78.mlp.experts.154.gate_proj", "model.layers.78.mlp.experts.155.gate_proj", "model.layers.78.mlp.experts.156.gate_proj", "model.layers.78.mlp.experts.157.gate_proj", "model.layers.78.mlp.experts.158.gate_proj", "model.layers.78.mlp.experts.159.gate_proj", "model.layers.78.mlp.experts.0.up_proj", "model.layers.78.mlp.experts.1.up_proj", "model.layers.78.mlp.experts.2.up_proj", "model.layers.78.mlp.experts.3.up_proj", "model.layers.78.mlp.experts.4.up_proj", "model.layers.78.mlp.experts.5.up_proj", "model.layers.78.mlp.experts.6.up_proj", "model.layers.78.mlp.experts.7.up_proj", "model.layers.78.mlp.experts.8.up_proj", "model.layers.78.mlp.experts.9.up_proj", "model.layers.78.mlp.experts.10.up_proj", "model.layers.78.mlp.experts.11.up_proj", "model.layers.78.mlp.experts.12.up_proj", "model.layers.78.mlp.experts.13.up_proj", "model.layers.78.mlp.experts.14.up_proj", "model.layers.78.mlp.experts.15.up_proj", "model.layers.78.mlp.experts.16.up_proj", "model.layers.78.mlp.experts.17.up_proj", "model.layers.78.mlp.experts.18.up_proj", "model.layers.78.mlp.experts.19.up_proj", "model.layers.78.mlp.experts.20.up_proj", "model.layers.78.mlp.experts.21.up_proj", "model.layers.78.mlp.experts.22.up_proj", "model.layers.78.mlp.experts.23.up_proj", "model.layers.78.mlp.experts.24.up_proj", "model.layers.78.mlp.experts.25.up_proj", "model.layers.78.mlp.experts.26.up_proj", "model.layers.78.mlp.experts.27.up_proj", "model.layers.78.mlp.experts.28.up_proj", "model.layers.78.mlp.experts.29.up_proj", "model.layers.78.mlp.experts.30.up_proj", "model.layers.78.mlp.experts.31.up_proj", "model.layers.78.mlp.experts.32.up_proj", "model.layers.78.mlp.experts.33.up_proj", "model.layers.78.mlp.experts.34.up_proj", "model.layers.78.mlp.experts.35.up_proj", "model.layers.78.mlp.experts.36.up_proj", "model.layers.78.mlp.experts.37.up_proj", "model.layers.78.mlp.experts.38.up_proj", "model.layers.78.mlp.experts.39.up_proj", "model.layers.78.mlp.experts.40.up_proj", "model.layers.78.mlp.experts.41.up_proj", "model.layers.78.mlp.experts.42.up_proj", "model.layers.78.mlp.experts.43.up_proj", "model.layers.78.mlp.experts.44.up_proj", "model.layers.78.mlp.experts.45.up_proj", "model.layers.78.mlp.experts.46.up_proj", "model.layers.78.mlp.experts.47.up_proj", "model.layers.78.mlp.experts.48.up_proj", "model.layers.78.mlp.experts.49.up_proj", "model.layers.78.mlp.experts.50.up_proj", "model.layers.78.mlp.experts.51.up_proj", "model.layers.78.mlp.experts.52.up_proj", "model.layers.78.mlp.experts.53.up_proj", "model.layers.78.mlp.experts.54.up_proj", "model.layers.78.mlp.experts.55.up_proj", "model.layers.78.mlp.experts.56.up_proj", "model.layers.78.mlp.experts.57.up_proj", "model.layers.78.mlp.experts.58.up_proj", "model.layers.78.mlp.experts.59.up_proj", "model.layers.78.mlp.experts.60.up_proj", "model.layers.78.mlp.experts.61.up_proj", "model.layers.78.mlp.experts.62.up_proj", "model.layers.78.mlp.experts.63.up_proj", "model.layers.78.mlp.experts.64.up_proj", "model.layers.78.mlp.experts.65.up_proj", "model.layers.78.mlp.experts.66.up_proj", "model.layers.78.mlp.experts.67.up_proj", "model.layers.78.mlp.experts.68.up_proj", "model.layers.78.mlp.experts.69.up_proj", "model.layers.78.mlp.experts.70.up_proj", "model.layers.78.mlp.experts.71.up_proj", "model.layers.78.mlp.experts.72.up_proj", "model.layers.78.mlp.experts.73.up_proj", "model.layers.78.mlp.experts.74.up_proj", "model.layers.78.mlp.experts.75.up_proj", "model.layers.78.mlp.experts.76.up_proj", "model.layers.78.mlp.experts.77.up_proj", "model.layers.78.mlp.experts.78.up_proj", "model.layers.78.mlp.experts.79.up_proj", "model.layers.78.mlp.experts.80.up_proj", "model.layers.78.mlp.experts.81.up_proj", "model.layers.78.mlp.experts.82.up_proj", "model.layers.78.mlp.experts.83.up_proj", "model.layers.78.mlp.experts.84.up_proj", "model.layers.78.mlp.experts.85.up_proj", "model.layers.78.mlp.experts.86.up_proj", "model.layers.78.mlp.experts.87.up_proj", "model.layers.78.mlp.experts.88.up_proj", "model.layers.78.mlp.experts.89.up_proj", "model.layers.78.mlp.experts.90.up_proj", "model.layers.78.mlp.experts.91.up_proj", "model.layers.78.mlp.experts.92.up_proj", "model.layers.78.mlp.experts.93.up_proj", "model.layers.78.mlp.experts.94.up_proj", "model.layers.78.mlp.experts.95.up_proj", "model.layers.78.mlp.experts.96.up_proj", "model.layers.78.mlp.experts.97.up_proj", "model.layers.78.mlp.experts.98.up_proj", "model.layers.78.mlp.experts.99.up_proj", "model.layers.78.mlp.experts.100.up_proj", "model.layers.78.mlp.experts.101.up_proj", "model.layers.78.mlp.experts.102.up_proj", "model.layers.78.mlp.experts.103.up_proj", "model.layers.78.mlp.experts.104.up_proj", "model.layers.78.mlp.experts.105.up_proj", "model.layers.78.mlp.experts.106.up_proj", "model.layers.78.mlp.experts.107.up_proj", "model.layers.78.mlp.experts.108.up_proj", "model.layers.78.mlp.experts.109.up_proj", "model.layers.78.mlp.experts.110.up_proj", "model.layers.78.mlp.experts.111.up_proj", "model.layers.78.mlp.experts.112.up_proj", "model.layers.78.mlp.experts.113.up_proj", "model.layers.78.mlp.experts.114.up_proj", "model.layers.78.mlp.experts.115.up_proj", "model.layers.78.mlp.experts.116.up_proj", "model.layers.78.mlp.experts.117.up_proj", "model.layers.78.mlp.experts.118.up_proj", "model.layers.78.mlp.experts.119.up_proj", "model.layers.78.mlp.experts.120.up_proj", "model.layers.78.mlp.experts.121.up_proj", "model.layers.78.mlp.experts.122.up_proj", "model.layers.78.mlp.experts.123.up_proj", "model.layers.78.mlp.experts.124.up_proj", "model.layers.78.mlp.experts.125.up_proj", "model.layers.78.mlp.experts.126.up_proj", "model.layers.78.mlp.experts.127.up_proj", "model.layers.78.mlp.experts.128.up_proj", "model.layers.78.mlp.experts.129.up_proj", "model.layers.78.mlp.experts.130.up_proj", "model.layers.78.mlp.experts.131.up_proj", "model.layers.78.mlp.experts.132.up_proj", "model.layers.78.mlp.experts.133.up_proj", "model.layers.78.mlp.experts.134.up_proj", "model.layers.78.mlp.experts.135.up_proj", "model.layers.78.mlp.experts.136.up_proj", "model.layers.78.mlp.experts.137.up_proj", "model.layers.78.mlp.experts.138.up_proj", "model.layers.78.mlp.experts.139.up_proj", "model.layers.78.mlp.experts.140.up_proj", "model.layers.78.mlp.experts.141.up_proj", "model.layers.78.mlp.experts.142.up_proj", "model.layers.78.mlp.experts.143.up_proj", "model.layers.78.mlp.experts.144.up_proj", "model.layers.78.mlp.experts.145.up_proj", "model.layers.78.mlp.experts.146.up_proj", "model.layers.78.mlp.experts.147.up_proj", "model.layers.78.mlp.experts.148.up_proj", "model.layers.78.mlp.experts.149.up_proj", "model.layers.78.mlp.experts.150.up_proj", "model.layers.78.mlp.experts.151.up_proj", "model.layers.78.mlp.experts.152.up_proj", "model.layers.78.mlp.experts.153.up_proj", "model.layers.78.mlp.experts.154.up_proj", "model.layers.78.mlp.experts.155.up_proj", "model.layers.78.mlp.experts.156.up_proj", "model.layers.78.mlp.experts.157.up_proj", "model.layers.78.mlp.experts.158.up_proj", "model.layers.78.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.961077891290231e-06, "dbits": 5033164800 } ] }, { "idx": 470, "layers": [ "model.layers.78.mlp.experts.0.down_proj", "model.layers.78.mlp.experts.1.down_proj", "model.layers.78.mlp.experts.2.down_proj", "model.layers.78.mlp.experts.3.down_proj", "model.layers.78.mlp.experts.4.down_proj", "model.layers.78.mlp.experts.5.down_proj", "model.layers.78.mlp.experts.6.down_proj", "model.layers.78.mlp.experts.7.down_proj", "model.layers.78.mlp.experts.8.down_proj", "model.layers.78.mlp.experts.9.down_proj", "model.layers.78.mlp.experts.10.down_proj", "model.layers.78.mlp.experts.11.down_proj", "model.layers.78.mlp.experts.12.down_proj", "model.layers.78.mlp.experts.13.down_proj", "model.layers.78.mlp.experts.14.down_proj", "model.layers.78.mlp.experts.15.down_proj", "model.layers.78.mlp.experts.16.down_proj", "model.layers.78.mlp.experts.17.down_proj", "model.layers.78.mlp.experts.18.down_proj", "model.layers.78.mlp.experts.19.down_proj", "model.layers.78.mlp.experts.20.down_proj", "model.layers.78.mlp.experts.21.down_proj", "model.layers.78.mlp.experts.22.down_proj", "model.layers.78.mlp.experts.23.down_proj", "model.layers.78.mlp.experts.24.down_proj", "model.layers.78.mlp.experts.25.down_proj", "model.layers.78.mlp.experts.26.down_proj", "model.layers.78.mlp.experts.27.down_proj", "model.layers.78.mlp.experts.28.down_proj", "model.layers.78.mlp.experts.29.down_proj", "model.layers.78.mlp.experts.30.down_proj", "model.layers.78.mlp.experts.31.down_proj", "model.layers.78.mlp.experts.32.down_proj", "model.layers.78.mlp.experts.33.down_proj", "model.layers.78.mlp.experts.34.down_proj", "model.layers.78.mlp.experts.35.down_proj", "model.layers.78.mlp.experts.36.down_proj", "model.layers.78.mlp.experts.37.down_proj", "model.layers.78.mlp.experts.38.down_proj", "model.layers.78.mlp.experts.39.down_proj", "model.layers.78.mlp.experts.40.down_proj", "model.layers.78.mlp.experts.41.down_proj", "model.layers.78.mlp.experts.42.down_proj", "model.layers.78.mlp.experts.43.down_proj", "model.layers.78.mlp.experts.44.down_proj", "model.layers.78.mlp.experts.45.down_proj", "model.layers.78.mlp.experts.46.down_proj", "model.layers.78.mlp.experts.47.down_proj", "model.layers.78.mlp.experts.48.down_proj", "model.layers.78.mlp.experts.49.down_proj", "model.layers.78.mlp.experts.50.down_proj", "model.layers.78.mlp.experts.51.down_proj", "model.layers.78.mlp.experts.52.down_proj", "model.layers.78.mlp.experts.53.down_proj", "model.layers.78.mlp.experts.54.down_proj", "model.layers.78.mlp.experts.55.down_proj", "model.layers.78.mlp.experts.56.down_proj", "model.layers.78.mlp.experts.57.down_proj", "model.layers.78.mlp.experts.58.down_proj", "model.layers.78.mlp.experts.59.down_proj", "model.layers.78.mlp.experts.60.down_proj", "model.layers.78.mlp.experts.61.down_proj", "model.layers.78.mlp.experts.62.down_proj", "model.layers.78.mlp.experts.63.down_proj", "model.layers.78.mlp.experts.64.down_proj", "model.layers.78.mlp.experts.65.down_proj", "model.layers.78.mlp.experts.66.down_proj", "model.layers.78.mlp.experts.67.down_proj", "model.layers.78.mlp.experts.68.down_proj", "model.layers.78.mlp.experts.69.down_proj", "model.layers.78.mlp.experts.70.down_proj", "model.layers.78.mlp.experts.71.down_proj", "model.layers.78.mlp.experts.72.down_proj", "model.layers.78.mlp.experts.73.down_proj", "model.layers.78.mlp.experts.74.down_proj", "model.layers.78.mlp.experts.75.down_proj", "model.layers.78.mlp.experts.76.down_proj", "model.layers.78.mlp.experts.77.down_proj", "model.layers.78.mlp.experts.78.down_proj", "model.layers.78.mlp.experts.79.down_proj", "model.layers.78.mlp.experts.80.down_proj", "model.layers.78.mlp.experts.81.down_proj", "model.layers.78.mlp.experts.82.down_proj", "model.layers.78.mlp.experts.83.down_proj", "model.layers.78.mlp.experts.84.down_proj", "model.layers.78.mlp.experts.85.down_proj", "model.layers.78.mlp.experts.86.down_proj", "model.layers.78.mlp.experts.87.down_proj", "model.layers.78.mlp.experts.88.down_proj", "model.layers.78.mlp.experts.89.down_proj", "model.layers.78.mlp.experts.90.down_proj", "model.layers.78.mlp.experts.91.down_proj", "model.layers.78.mlp.experts.92.down_proj", "model.layers.78.mlp.experts.93.down_proj", "model.layers.78.mlp.experts.94.down_proj", "model.layers.78.mlp.experts.95.down_proj", "model.layers.78.mlp.experts.96.down_proj", "model.layers.78.mlp.experts.97.down_proj", "model.layers.78.mlp.experts.98.down_proj", "model.layers.78.mlp.experts.99.down_proj", "model.layers.78.mlp.experts.100.down_proj", "model.layers.78.mlp.experts.101.down_proj", "model.layers.78.mlp.experts.102.down_proj", "model.layers.78.mlp.experts.103.down_proj", "model.layers.78.mlp.experts.104.down_proj", "model.layers.78.mlp.experts.105.down_proj", "model.layers.78.mlp.experts.106.down_proj", "model.layers.78.mlp.experts.107.down_proj", "model.layers.78.mlp.experts.108.down_proj", "model.layers.78.mlp.experts.109.down_proj", "model.layers.78.mlp.experts.110.down_proj", "model.layers.78.mlp.experts.111.down_proj", "model.layers.78.mlp.experts.112.down_proj", "model.layers.78.mlp.experts.113.down_proj", "model.layers.78.mlp.experts.114.down_proj", "model.layers.78.mlp.experts.115.down_proj", "model.layers.78.mlp.experts.116.down_proj", "model.layers.78.mlp.experts.117.down_proj", "model.layers.78.mlp.experts.118.down_proj", "model.layers.78.mlp.experts.119.down_proj", "model.layers.78.mlp.experts.120.down_proj", "model.layers.78.mlp.experts.121.down_proj", "model.layers.78.mlp.experts.122.down_proj", "model.layers.78.mlp.experts.123.down_proj", "model.layers.78.mlp.experts.124.down_proj", "model.layers.78.mlp.experts.125.down_proj", "model.layers.78.mlp.experts.126.down_proj", "model.layers.78.mlp.experts.127.down_proj", "model.layers.78.mlp.experts.128.down_proj", "model.layers.78.mlp.experts.129.down_proj", "model.layers.78.mlp.experts.130.down_proj", "model.layers.78.mlp.experts.131.down_proj", "model.layers.78.mlp.experts.132.down_proj", "model.layers.78.mlp.experts.133.down_proj", "model.layers.78.mlp.experts.134.down_proj", "model.layers.78.mlp.experts.135.down_proj", "model.layers.78.mlp.experts.136.down_proj", "model.layers.78.mlp.experts.137.down_proj", "model.layers.78.mlp.experts.138.down_proj", "model.layers.78.mlp.experts.139.down_proj", "model.layers.78.mlp.experts.140.down_proj", "model.layers.78.mlp.experts.141.down_proj", "model.layers.78.mlp.experts.142.down_proj", "model.layers.78.mlp.experts.143.down_proj", "model.layers.78.mlp.experts.144.down_proj", "model.layers.78.mlp.experts.145.down_proj", "model.layers.78.mlp.experts.146.down_proj", "model.layers.78.mlp.experts.147.down_proj", "model.layers.78.mlp.experts.148.down_proj", "model.layers.78.mlp.experts.149.down_proj", "model.layers.78.mlp.experts.150.down_proj", "model.layers.78.mlp.experts.151.down_proj", "model.layers.78.mlp.experts.152.down_proj", "model.layers.78.mlp.experts.153.down_proj", "model.layers.78.mlp.experts.154.down_proj", "model.layers.78.mlp.experts.155.down_proj", "model.layers.78.mlp.experts.156.down_proj", "model.layers.78.mlp.experts.157.down_proj", "model.layers.78.mlp.experts.158.down_proj", "model.layers.78.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.856065450236158e-06, "dbits": 2516582400 } ] }, { "idx": 471, "layers": [ "model.layers.79.self_attn.q_proj" ], "candidates": [ { "dkld": 2.0968378521502235e-06, "dbits": 125829120 } ] }, { "idx": 472, "layers": [ "model.layers.79.self_attn.k_proj", "model.layers.79.self_attn.v_proj" ], "candidates": [ { "dkld": -3.0098308343440294e-06, "dbits": 20971520 } ] }, { "idx": 473, "layers": [ "model.layers.79.self_attn.o_proj" ], "candidates": [ { "dkld": -1.5366415027529435e-06, "dbits": 125829120 } ] }, { "idx": 474, "layers": [ "model.layers.79.mlp.shared_experts.gate_proj", "model.layers.79.mlp.shared_experts.up_proj", "model.layers.79.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 4.117522621527314e-06, "dbits": 47185920 } ] }, { "idx": 475, "layers": [ "model.layers.79.mlp.experts.0.gate_proj", "model.layers.79.mlp.experts.1.gate_proj", "model.layers.79.mlp.experts.2.gate_proj", "model.layers.79.mlp.experts.3.gate_proj", "model.layers.79.mlp.experts.4.gate_proj", "model.layers.79.mlp.experts.5.gate_proj", "model.layers.79.mlp.experts.6.gate_proj", "model.layers.79.mlp.experts.7.gate_proj", "model.layers.79.mlp.experts.8.gate_proj", "model.layers.79.mlp.experts.9.gate_proj", "model.layers.79.mlp.experts.10.gate_proj", "model.layers.79.mlp.experts.11.gate_proj", "model.layers.79.mlp.experts.12.gate_proj", "model.layers.79.mlp.experts.13.gate_proj", "model.layers.79.mlp.experts.14.gate_proj", "model.layers.79.mlp.experts.15.gate_proj", "model.layers.79.mlp.experts.16.gate_proj", "model.layers.79.mlp.experts.17.gate_proj", "model.layers.79.mlp.experts.18.gate_proj", "model.layers.79.mlp.experts.19.gate_proj", "model.layers.79.mlp.experts.20.gate_proj", "model.layers.79.mlp.experts.21.gate_proj", "model.layers.79.mlp.experts.22.gate_proj", "model.layers.79.mlp.experts.23.gate_proj", "model.layers.79.mlp.experts.24.gate_proj", "model.layers.79.mlp.experts.25.gate_proj", "model.layers.79.mlp.experts.26.gate_proj", "model.layers.79.mlp.experts.27.gate_proj", "model.layers.79.mlp.experts.28.gate_proj", "model.layers.79.mlp.experts.29.gate_proj", "model.layers.79.mlp.experts.30.gate_proj", "model.layers.79.mlp.experts.31.gate_proj", "model.layers.79.mlp.experts.32.gate_proj", "model.layers.79.mlp.experts.33.gate_proj", "model.layers.79.mlp.experts.34.gate_proj", "model.layers.79.mlp.experts.35.gate_proj", "model.layers.79.mlp.experts.36.gate_proj", "model.layers.79.mlp.experts.37.gate_proj", "model.layers.79.mlp.experts.38.gate_proj", "model.layers.79.mlp.experts.39.gate_proj", "model.layers.79.mlp.experts.40.gate_proj", "model.layers.79.mlp.experts.41.gate_proj", "model.layers.79.mlp.experts.42.gate_proj", "model.layers.79.mlp.experts.43.gate_proj", "model.layers.79.mlp.experts.44.gate_proj", "model.layers.79.mlp.experts.45.gate_proj", "model.layers.79.mlp.experts.46.gate_proj", "model.layers.79.mlp.experts.47.gate_proj", "model.layers.79.mlp.experts.48.gate_proj", "model.layers.79.mlp.experts.49.gate_proj", "model.layers.79.mlp.experts.50.gate_proj", "model.layers.79.mlp.experts.51.gate_proj", "model.layers.79.mlp.experts.52.gate_proj", "model.layers.79.mlp.experts.53.gate_proj", "model.layers.79.mlp.experts.54.gate_proj", "model.layers.79.mlp.experts.55.gate_proj", "model.layers.79.mlp.experts.56.gate_proj", "model.layers.79.mlp.experts.57.gate_proj", "model.layers.79.mlp.experts.58.gate_proj", "model.layers.79.mlp.experts.59.gate_proj", "model.layers.79.mlp.experts.60.gate_proj", "model.layers.79.mlp.experts.61.gate_proj", "model.layers.79.mlp.experts.62.gate_proj", "model.layers.79.mlp.experts.63.gate_proj", "model.layers.79.mlp.experts.64.gate_proj", "model.layers.79.mlp.experts.65.gate_proj", "model.layers.79.mlp.experts.66.gate_proj", "model.layers.79.mlp.experts.67.gate_proj", "model.layers.79.mlp.experts.68.gate_proj", "model.layers.79.mlp.experts.69.gate_proj", "model.layers.79.mlp.experts.70.gate_proj", "model.layers.79.mlp.experts.71.gate_proj", "model.layers.79.mlp.experts.72.gate_proj", "model.layers.79.mlp.experts.73.gate_proj", "model.layers.79.mlp.experts.74.gate_proj", "model.layers.79.mlp.experts.75.gate_proj", "model.layers.79.mlp.experts.76.gate_proj", "model.layers.79.mlp.experts.77.gate_proj", "model.layers.79.mlp.experts.78.gate_proj", "model.layers.79.mlp.experts.79.gate_proj", "model.layers.79.mlp.experts.80.gate_proj", "model.layers.79.mlp.experts.81.gate_proj", "model.layers.79.mlp.experts.82.gate_proj", "model.layers.79.mlp.experts.83.gate_proj", "model.layers.79.mlp.experts.84.gate_proj", "model.layers.79.mlp.experts.85.gate_proj", "model.layers.79.mlp.experts.86.gate_proj", "model.layers.79.mlp.experts.87.gate_proj", "model.layers.79.mlp.experts.88.gate_proj", "model.layers.79.mlp.experts.89.gate_proj", "model.layers.79.mlp.experts.90.gate_proj", "model.layers.79.mlp.experts.91.gate_proj", "model.layers.79.mlp.experts.92.gate_proj", "model.layers.79.mlp.experts.93.gate_proj", "model.layers.79.mlp.experts.94.gate_proj", "model.layers.79.mlp.experts.95.gate_proj", "model.layers.79.mlp.experts.96.gate_proj", "model.layers.79.mlp.experts.97.gate_proj", "model.layers.79.mlp.experts.98.gate_proj", "model.layers.79.mlp.experts.99.gate_proj", "model.layers.79.mlp.experts.100.gate_proj", "model.layers.79.mlp.experts.101.gate_proj", "model.layers.79.mlp.experts.102.gate_proj", "model.layers.79.mlp.experts.103.gate_proj", "model.layers.79.mlp.experts.104.gate_proj", "model.layers.79.mlp.experts.105.gate_proj", "model.layers.79.mlp.experts.106.gate_proj", "model.layers.79.mlp.experts.107.gate_proj", "model.layers.79.mlp.experts.108.gate_proj", "model.layers.79.mlp.experts.109.gate_proj", "model.layers.79.mlp.experts.110.gate_proj", "model.layers.79.mlp.experts.111.gate_proj", "model.layers.79.mlp.experts.112.gate_proj", "model.layers.79.mlp.experts.113.gate_proj", "model.layers.79.mlp.experts.114.gate_proj", "model.layers.79.mlp.experts.115.gate_proj", "model.layers.79.mlp.experts.116.gate_proj", "model.layers.79.mlp.experts.117.gate_proj", "model.layers.79.mlp.experts.118.gate_proj", "model.layers.79.mlp.experts.119.gate_proj", "model.layers.79.mlp.experts.120.gate_proj", "model.layers.79.mlp.experts.121.gate_proj", "model.layers.79.mlp.experts.122.gate_proj", "model.layers.79.mlp.experts.123.gate_proj", "model.layers.79.mlp.experts.124.gate_proj", "model.layers.79.mlp.experts.125.gate_proj", "model.layers.79.mlp.experts.126.gate_proj", "model.layers.79.mlp.experts.127.gate_proj", "model.layers.79.mlp.experts.128.gate_proj", "model.layers.79.mlp.experts.129.gate_proj", "model.layers.79.mlp.experts.130.gate_proj", "model.layers.79.mlp.experts.131.gate_proj", "model.layers.79.mlp.experts.132.gate_proj", "model.layers.79.mlp.experts.133.gate_proj", "model.layers.79.mlp.experts.134.gate_proj", "model.layers.79.mlp.experts.135.gate_proj", "model.layers.79.mlp.experts.136.gate_proj", "model.layers.79.mlp.experts.137.gate_proj", "model.layers.79.mlp.experts.138.gate_proj", "model.layers.79.mlp.experts.139.gate_proj", "model.layers.79.mlp.experts.140.gate_proj", "model.layers.79.mlp.experts.141.gate_proj", "model.layers.79.mlp.experts.142.gate_proj", "model.layers.79.mlp.experts.143.gate_proj", "model.layers.79.mlp.experts.144.gate_proj", "model.layers.79.mlp.experts.145.gate_proj", "model.layers.79.mlp.experts.146.gate_proj", "model.layers.79.mlp.experts.147.gate_proj", "model.layers.79.mlp.experts.148.gate_proj", "model.layers.79.mlp.experts.149.gate_proj", "model.layers.79.mlp.experts.150.gate_proj", "model.layers.79.mlp.experts.151.gate_proj", "model.layers.79.mlp.experts.152.gate_proj", "model.layers.79.mlp.experts.153.gate_proj", "model.layers.79.mlp.experts.154.gate_proj", "model.layers.79.mlp.experts.155.gate_proj", "model.layers.79.mlp.experts.156.gate_proj", "model.layers.79.mlp.experts.157.gate_proj", "model.layers.79.mlp.experts.158.gate_proj", "model.layers.79.mlp.experts.159.gate_proj", "model.layers.79.mlp.experts.0.up_proj", "model.layers.79.mlp.experts.1.up_proj", "model.layers.79.mlp.experts.2.up_proj", "model.layers.79.mlp.experts.3.up_proj", "model.layers.79.mlp.experts.4.up_proj", "model.layers.79.mlp.experts.5.up_proj", "model.layers.79.mlp.experts.6.up_proj", "model.layers.79.mlp.experts.7.up_proj", "model.layers.79.mlp.experts.8.up_proj", "model.layers.79.mlp.experts.9.up_proj", "model.layers.79.mlp.experts.10.up_proj", "model.layers.79.mlp.experts.11.up_proj", "model.layers.79.mlp.experts.12.up_proj", "model.layers.79.mlp.experts.13.up_proj", "model.layers.79.mlp.experts.14.up_proj", "model.layers.79.mlp.experts.15.up_proj", "model.layers.79.mlp.experts.16.up_proj", "model.layers.79.mlp.experts.17.up_proj", "model.layers.79.mlp.experts.18.up_proj", "model.layers.79.mlp.experts.19.up_proj", "model.layers.79.mlp.experts.20.up_proj", "model.layers.79.mlp.experts.21.up_proj", "model.layers.79.mlp.experts.22.up_proj", "model.layers.79.mlp.experts.23.up_proj", "model.layers.79.mlp.experts.24.up_proj", "model.layers.79.mlp.experts.25.up_proj", "model.layers.79.mlp.experts.26.up_proj", "model.layers.79.mlp.experts.27.up_proj", "model.layers.79.mlp.experts.28.up_proj", "model.layers.79.mlp.experts.29.up_proj", "model.layers.79.mlp.experts.30.up_proj", "model.layers.79.mlp.experts.31.up_proj", "model.layers.79.mlp.experts.32.up_proj", "model.layers.79.mlp.experts.33.up_proj", "model.layers.79.mlp.experts.34.up_proj", "model.layers.79.mlp.experts.35.up_proj", "model.layers.79.mlp.experts.36.up_proj", "model.layers.79.mlp.experts.37.up_proj", "model.layers.79.mlp.experts.38.up_proj", "model.layers.79.mlp.experts.39.up_proj", "model.layers.79.mlp.experts.40.up_proj", "model.layers.79.mlp.experts.41.up_proj", "model.layers.79.mlp.experts.42.up_proj", "model.layers.79.mlp.experts.43.up_proj", "model.layers.79.mlp.experts.44.up_proj", "model.layers.79.mlp.experts.45.up_proj", "model.layers.79.mlp.experts.46.up_proj", "model.layers.79.mlp.experts.47.up_proj", "model.layers.79.mlp.experts.48.up_proj", "model.layers.79.mlp.experts.49.up_proj", "model.layers.79.mlp.experts.50.up_proj", "model.layers.79.mlp.experts.51.up_proj", "model.layers.79.mlp.experts.52.up_proj", "model.layers.79.mlp.experts.53.up_proj", "model.layers.79.mlp.experts.54.up_proj", "model.layers.79.mlp.experts.55.up_proj", "model.layers.79.mlp.experts.56.up_proj", "model.layers.79.mlp.experts.57.up_proj", "model.layers.79.mlp.experts.58.up_proj", "model.layers.79.mlp.experts.59.up_proj", "model.layers.79.mlp.experts.60.up_proj", "model.layers.79.mlp.experts.61.up_proj", "model.layers.79.mlp.experts.62.up_proj", "model.layers.79.mlp.experts.63.up_proj", "model.layers.79.mlp.experts.64.up_proj", "model.layers.79.mlp.experts.65.up_proj", "model.layers.79.mlp.experts.66.up_proj", "model.layers.79.mlp.experts.67.up_proj", "model.layers.79.mlp.experts.68.up_proj", "model.layers.79.mlp.experts.69.up_proj", "model.layers.79.mlp.experts.70.up_proj", "model.layers.79.mlp.experts.71.up_proj", "model.layers.79.mlp.experts.72.up_proj", "model.layers.79.mlp.experts.73.up_proj", "model.layers.79.mlp.experts.74.up_proj", "model.layers.79.mlp.experts.75.up_proj", "model.layers.79.mlp.experts.76.up_proj", "model.layers.79.mlp.experts.77.up_proj", "model.layers.79.mlp.experts.78.up_proj", "model.layers.79.mlp.experts.79.up_proj", "model.layers.79.mlp.experts.80.up_proj", "model.layers.79.mlp.experts.81.up_proj", "model.layers.79.mlp.experts.82.up_proj", "model.layers.79.mlp.experts.83.up_proj", "model.layers.79.mlp.experts.84.up_proj", "model.layers.79.mlp.experts.85.up_proj", "model.layers.79.mlp.experts.86.up_proj", "model.layers.79.mlp.experts.87.up_proj", "model.layers.79.mlp.experts.88.up_proj", "model.layers.79.mlp.experts.89.up_proj", "model.layers.79.mlp.experts.90.up_proj", "model.layers.79.mlp.experts.91.up_proj", "model.layers.79.mlp.experts.92.up_proj", "model.layers.79.mlp.experts.93.up_proj", "model.layers.79.mlp.experts.94.up_proj", "model.layers.79.mlp.experts.95.up_proj", "model.layers.79.mlp.experts.96.up_proj", "model.layers.79.mlp.experts.97.up_proj", "model.layers.79.mlp.experts.98.up_proj", "model.layers.79.mlp.experts.99.up_proj", "model.layers.79.mlp.experts.100.up_proj", "model.layers.79.mlp.experts.101.up_proj", "model.layers.79.mlp.experts.102.up_proj", "model.layers.79.mlp.experts.103.up_proj", "model.layers.79.mlp.experts.104.up_proj", "model.layers.79.mlp.experts.105.up_proj", "model.layers.79.mlp.experts.106.up_proj", "model.layers.79.mlp.experts.107.up_proj", "model.layers.79.mlp.experts.108.up_proj", "model.layers.79.mlp.experts.109.up_proj", "model.layers.79.mlp.experts.110.up_proj", "model.layers.79.mlp.experts.111.up_proj", "model.layers.79.mlp.experts.112.up_proj", "model.layers.79.mlp.experts.113.up_proj", "model.layers.79.mlp.experts.114.up_proj", "model.layers.79.mlp.experts.115.up_proj", "model.layers.79.mlp.experts.116.up_proj", "model.layers.79.mlp.experts.117.up_proj", "model.layers.79.mlp.experts.118.up_proj", "model.layers.79.mlp.experts.119.up_proj", "model.layers.79.mlp.experts.120.up_proj", "model.layers.79.mlp.experts.121.up_proj", "model.layers.79.mlp.experts.122.up_proj", "model.layers.79.mlp.experts.123.up_proj", "model.layers.79.mlp.experts.124.up_proj", "model.layers.79.mlp.experts.125.up_proj", "model.layers.79.mlp.experts.126.up_proj", "model.layers.79.mlp.experts.127.up_proj", "model.layers.79.mlp.experts.128.up_proj", "model.layers.79.mlp.experts.129.up_proj", "model.layers.79.mlp.experts.130.up_proj", "model.layers.79.mlp.experts.131.up_proj", "model.layers.79.mlp.experts.132.up_proj", "model.layers.79.mlp.experts.133.up_proj", "model.layers.79.mlp.experts.134.up_proj", "model.layers.79.mlp.experts.135.up_proj", "model.layers.79.mlp.experts.136.up_proj", "model.layers.79.mlp.experts.137.up_proj", "model.layers.79.mlp.experts.138.up_proj", "model.layers.79.mlp.experts.139.up_proj", "model.layers.79.mlp.experts.140.up_proj", "model.layers.79.mlp.experts.141.up_proj", "model.layers.79.mlp.experts.142.up_proj", "model.layers.79.mlp.experts.143.up_proj", "model.layers.79.mlp.experts.144.up_proj", "model.layers.79.mlp.experts.145.up_proj", "model.layers.79.mlp.experts.146.up_proj", "model.layers.79.mlp.experts.147.up_proj", "model.layers.79.mlp.experts.148.up_proj", "model.layers.79.mlp.experts.149.up_proj", "model.layers.79.mlp.experts.150.up_proj", "model.layers.79.mlp.experts.151.up_proj", "model.layers.79.mlp.experts.152.up_proj", "model.layers.79.mlp.experts.153.up_proj", "model.layers.79.mlp.experts.154.up_proj", "model.layers.79.mlp.experts.155.up_proj", "model.layers.79.mlp.experts.156.up_proj", "model.layers.79.mlp.experts.157.up_proj", "model.layers.79.mlp.experts.158.up_proj", "model.layers.79.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 3.3543532481416843e-06, "dbits": 5033164800 } ] }, { "idx": 476, "layers": [ "model.layers.79.mlp.experts.0.down_proj", "model.layers.79.mlp.experts.1.down_proj", "model.layers.79.mlp.experts.2.down_proj", "model.layers.79.mlp.experts.3.down_proj", "model.layers.79.mlp.experts.4.down_proj", "model.layers.79.mlp.experts.5.down_proj", "model.layers.79.mlp.experts.6.down_proj", "model.layers.79.mlp.experts.7.down_proj", "model.layers.79.mlp.experts.8.down_proj", "model.layers.79.mlp.experts.9.down_proj", "model.layers.79.mlp.experts.10.down_proj", "model.layers.79.mlp.experts.11.down_proj", "model.layers.79.mlp.experts.12.down_proj", "model.layers.79.mlp.experts.13.down_proj", "model.layers.79.mlp.experts.14.down_proj", "model.layers.79.mlp.experts.15.down_proj", "model.layers.79.mlp.experts.16.down_proj", "model.layers.79.mlp.experts.17.down_proj", "model.layers.79.mlp.experts.18.down_proj", "model.layers.79.mlp.experts.19.down_proj", "model.layers.79.mlp.experts.20.down_proj", "model.layers.79.mlp.experts.21.down_proj", "model.layers.79.mlp.experts.22.down_proj", "model.layers.79.mlp.experts.23.down_proj", "model.layers.79.mlp.experts.24.down_proj", "model.layers.79.mlp.experts.25.down_proj", "model.layers.79.mlp.experts.26.down_proj", "model.layers.79.mlp.experts.27.down_proj", "model.layers.79.mlp.experts.28.down_proj", "model.layers.79.mlp.experts.29.down_proj", "model.layers.79.mlp.experts.30.down_proj", "model.layers.79.mlp.experts.31.down_proj", "model.layers.79.mlp.experts.32.down_proj", "model.layers.79.mlp.experts.33.down_proj", "model.layers.79.mlp.experts.34.down_proj", "model.layers.79.mlp.experts.35.down_proj", "model.layers.79.mlp.experts.36.down_proj", "model.layers.79.mlp.experts.37.down_proj", "model.layers.79.mlp.experts.38.down_proj", "model.layers.79.mlp.experts.39.down_proj", "model.layers.79.mlp.experts.40.down_proj", "model.layers.79.mlp.experts.41.down_proj", "model.layers.79.mlp.experts.42.down_proj", "model.layers.79.mlp.experts.43.down_proj", "model.layers.79.mlp.experts.44.down_proj", "model.layers.79.mlp.experts.45.down_proj", "model.layers.79.mlp.experts.46.down_proj", "model.layers.79.mlp.experts.47.down_proj", "model.layers.79.mlp.experts.48.down_proj", "model.layers.79.mlp.experts.49.down_proj", "model.layers.79.mlp.experts.50.down_proj", "model.layers.79.mlp.experts.51.down_proj", "model.layers.79.mlp.experts.52.down_proj", "model.layers.79.mlp.experts.53.down_proj", "model.layers.79.mlp.experts.54.down_proj", "model.layers.79.mlp.experts.55.down_proj", "model.layers.79.mlp.experts.56.down_proj", "model.layers.79.mlp.experts.57.down_proj", "model.layers.79.mlp.experts.58.down_proj", "model.layers.79.mlp.experts.59.down_proj", "model.layers.79.mlp.experts.60.down_proj", "model.layers.79.mlp.experts.61.down_proj", "model.layers.79.mlp.experts.62.down_proj", "model.layers.79.mlp.experts.63.down_proj", "model.layers.79.mlp.experts.64.down_proj", "model.layers.79.mlp.experts.65.down_proj", "model.layers.79.mlp.experts.66.down_proj", "model.layers.79.mlp.experts.67.down_proj", "model.layers.79.mlp.experts.68.down_proj", "model.layers.79.mlp.experts.69.down_proj", "model.layers.79.mlp.experts.70.down_proj", "model.layers.79.mlp.experts.71.down_proj", "model.layers.79.mlp.experts.72.down_proj", "model.layers.79.mlp.experts.73.down_proj", "model.layers.79.mlp.experts.74.down_proj", "model.layers.79.mlp.experts.75.down_proj", "model.layers.79.mlp.experts.76.down_proj", "model.layers.79.mlp.experts.77.down_proj", "model.layers.79.mlp.experts.78.down_proj", "model.layers.79.mlp.experts.79.down_proj", "model.layers.79.mlp.experts.80.down_proj", "model.layers.79.mlp.experts.81.down_proj", "model.layers.79.mlp.experts.82.down_proj", "model.layers.79.mlp.experts.83.down_proj", "model.layers.79.mlp.experts.84.down_proj", "model.layers.79.mlp.experts.85.down_proj", "model.layers.79.mlp.experts.86.down_proj", "model.layers.79.mlp.experts.87.down_proj", "model.layers.79.mlp.experts.88.down_proj", "model.layers.79.mlp.experts.89.down_proj", "model.layers.79.mlp.experts.90.down_proj", "model.layers.79.mlp.experts.91.down_proj", "model.layers.79.mlp.experts.92.down_proj", "model.layers.79.mlp.experts.93.down_proj", "model.layers.79.mlp.experts.94.down_proj", "model.layers.79.mlp.experts.95.down_proj", "model.layers.79.mlp.experts.96.down_proj", "model.layers.79.mlp.experts.97.down_proj", "model.layers.79.mlp.experts.98.down_proj", "model.layers.79.mlp.experts.99.down_proj", "model.layers.79.mlp.experts.100.down_proj", "model.layers.79.mlp.experts.101.down_proj", "model.layers.79.mlp.experts.102.down_proj", "model.layers.79.mlp.experts.103.down_proj", "model.layers.79.mlp.experts.104.down_proj", "model.layers.79.mlp.experts.105.down_proj", "model.layers.79.mlp.experts.106.down_proj", "model.layers.79.mlp.experts.107.down_proj", "model.layers.79.mlp.experts.108.down_proj", "model.layers.79.mlp.experts.109.down_proj", "model.layers.79.mlp.experts.110.down_proj", "model.layers.79.mlp.experts.111.down_proj", "model.layers.79.mlp.experts.112.down_proj", "model.layers.79.mlp.experts.113.down_proj", "model.layers.79.mlp.experts.114.down_proj", "model.layers.79.mlp.experts.115.down_proj", "model.layers.79.mlp.experts.116.down_proj", "model.layers.79.mlp.experts.117.down_proj", "model.layers.79.mlp.experts.118.down_proj", "model.layers.79.mlp.experts.119.down_proj", "model.layers.79.mlp.experts.120.down_proj", "model.layers.79.mlp.experts.121.down_proj", "model.layers.79.mlp.experts.122.down_proj", "model.layers.79.mlp.experts.123.down_proj", "model.layers.79.mlp.experts.124.down_proj", "model.layers.79.mlp.experts.125.down_proj", "model.layers.79.mlp.experts.126.down_proj", "model.layers.79.mlp.experts.127.down_proj", "model.layers.79.mlp.experts.128.down_proj", "model.layers.79.mlp.experts.129.down_proj", "model.layers.79.mlp.experts.130.down_proj", "model.layers.79.mlp.experts.131.down_proj", "model.layers.79.mlp.experts.132.down_proj", "model.layers.79.mlp.experts.133.down_proj", "model.layers.79.mlp.experts.134.down_proj", "model.layers.79.mlp.experts.135.down_proj", "model.layers.79.mlp.experts.136.down_proj", "model.layers.79.mlp.experts.137.down_proj", "model.layers.79.mlp.experts.138.down_proj", "model.layers.79.mlp.experts.139.down_proj", "model.layers.79.mlp.experts.140.down_proj", "model.layers.79.mlp.experts.141.down_proj", "model.layers.79.mlp.experts.142.down_proj", "model.layers.79.mlp.experts.143.down_proj", "model.layers.79.mlp.experts.144.down_proj", "model.layers.79.mlp.experts.145.down_proj", "model.layers.79.mlp.experts.146.down_proj", "model.layers.79.mlp.experts.147.down_proj", "model.layers.79.mlp.experts.148.down_proj", "model.layers.79.mlp.experts.149.down_proj", "model.layers.79.mlp.experts.150.down_proj", "model.layers.79.mlp.experts.151.down_proj", "model.layers.79.mlp.experts.152.down_proj", "model.layers.79.mlp.experts.153.down_proj", "model.layers.79.mlp.experts.154.down_proj", "model.layers.79.mlp.experts.155.down_proj", "model.layers.79.mlp.experts.156.down_proj", "model.layers.79.mlp.experts.157.down_proj", "model.layers.79.mlp.experts.158.down_proj", "model.layers.79.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -6.932270480320291e-07, "dbits": 2516582400 } ] }, { "idx": 477, "layers": [ "model.layers.80.self_attn.q_proj" ], "candidates": [ { "dkld": -2.747419057413979e-06, "dbits": 125829120 } ] }, { "idx": 478, "layers": [ "model.layers.80.self_attn.k_proj", "model.layers.80.self_attn.v_proj" ], "candidates": [ { "dkld": -9.398179827257557e-07, "dbits": 20971520 } ] }, { "idx": 479, "layers": [ "model.layers.80.self_attn.o_proj" ], "candidates": [ { "dkld": -1.90984865184869e-06, "dbits": 125829120 } ] }, { "idx": 480, "layers": [ "model.layers.80.mlp.shared_experts.gate_proj", "model.layers.80.mlp.shared_experts.up_proj", "model.layers.80.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -8.16437532193961e-06, "dbits": 47185920 } ] }, { "idx": 481, "layers": [ "model.layers.80.mlp.experts.0.gate_proj", "model.layers.80.mlp.experts.1.gate_proj", "model.layers.80.mlp.experts.2.gate_proj", "model.layers.80.mlp.experts.3.gate_proj", "model.layers.80.mlp.experts.4.gate_proj", "model.layers.80.mlp.experts.5.gate_proj", "model.layers.80.mlp.experts.6.gate_proj", "model.layers.80.mlp.experts.7.gate_proj", "model.layers.80.mlp.experts.8.gate_proj", "model.layers.80.mlp.experts.9.gate_proj", "model.layers.80.mlp.experts.10.gate_proj", "model.layers.80.mlp.experts.11.gate_proj", "model.layers.80.mlp.experts.12.gate_proj", "model.layers.80.mlp.experts.13.gate_proj", "model.layers.80.mlp.experts.14.gate_proj", "model.layers.80.mlp.experts.15.gate_proj", "model.layers.80.mlp.experts.16.gate_proj", "model.layers.80.mlp.experts.17.gate_proj", "model.layers.80.mlp.experts.18.gate_proj", "model.layers.80.mlp.experts.19.gate_proj", "model.layers.80.mlp.experts.20.gate_proj", "model.layers.80.mlp.experts.21.gate_proj", "model.layers.80.mlp.experts.22.gate_proj", "model.layers.80.mlp.experts.23.gate_proj", "model.layers.80.mlp.experts.24.gate_proj", "model.layers.80.mlp.experts.25.gate_proj", "model.layers.80.mlp.experts.26.gate_proj", "model.layers.80.mlp.experts.27.gate_proj", "model.layers.80.mlp.experts.28.gate_proj", "model.layers.80.mlp.experts.29.gate_proj", "model.layers.80.mlp.experts.30.gate_proj", "model.layers.80.mlp.experts.31.gate_proj", "model.layers.80.mlp.experts.32.gate_proj", "model.layers.80.mlp.experts.33.gate_proj", "model.layers.80.mlp.experts.34.gate_proj", "model.layers.80.mlp.experts.35.gate_proj", "model.layers.80.mlp.experts.36.gate_proj", "model.layers.80.mlp.experts.37.gate_proj", "model.layers.80.mlp.experts.38.gate_proj", "model.layers.80.mlp.experts.39.gate_proj", "model.layers.80.mlp.experts.40.gate_proj", "model.layers.80.mlp.experts.41.gate_proj", "model.layers.80.mlp.experts.42.gate_proj", "model.layers.80.mlp.experts.43.gate_proj", "model.layers.80.mlp.experts.44.gate_proj", "model.layers.80.mlp.experts.45.gate_proj", "model.layers.80.mlp.experts.46.gate_proj", "model.layers.80.mlp.experts.47.gate_proj", "model.layers.80.mlp.experts.48.gate_proj", "model.layers.80.mlp.experts.49.gate_proj", "model.layers.80.mlp.experts.50.gate_proj", "model.layers.80.mlp.experts.51.gate_proj", "model.layers.80.mlp.experts.52.gate_proj", "model.layers.80.mlp.experts.53.gate_proj", "model.layers.80.mlp.experts.54.gate_proj", "model.layers.80.mlp.experts.55.gate_proj", "model.layers.80.mlp.experts.56.gate_proj", "model.layers.80.mlp.experts.57.gate_proj", "model.layers.80.mlp.experts.58.gate_proj", "model.layers.80.mlp.experts.59.gate_proj", "model.layers.80.mlp.experts.60.gate_proj", "model.layers.80.mlp.experts.61.gate_proj", "model.layers.80.mlp.experts.62.gate_proj", "model.layers.80.mlp.experts.63.gate_proj", "model.layers.80.mlp.experts.64.gate_proj", "model.layers.80.mlp.experts.65.gate_proj", "model.layers.80.mlp.experts.66.gate_proj", "model.layers.80.mlp.experts.67.gate_proj", "model.layers.80.mlp.experts.68.gate_proj", "model.layers.80.mlp.experts.69.gate_proj", "model.layers.80.mlp.experts.70.gate_proj", "model.layers.80.mlp.experts.71.gate_proj", "model.layers.80.mlp.experts.72.gate_proj", "model.layers.80.mlp.experts.73.gate_proj", "model.layers.80.mlp.experts.74.gate_proj", "model.layers.80.mlp.experts.75.gate_proj", "model.layers.80.mlp.experts.76.gate_proj", "model.layers.80.mlp.experts.77.gate_proj", "model.layers.80.mlp.experts.78.gate_proj", "model.layers.80.mlp.experts.79.gate_proj", "model.layers.80.mlp.experts.80.gate_proj", "model.layers.80.mlp.experts.81.gate_proj", "model.layers.80.mlp.experts.82.gate_proj", "model.layers.80.mlp.experts.83.gate_proj", "model.layers.80.mlp.experts.84.gate_proj", "model.layers.80.mlp.experts.85.gate_proj", "model.layers.80.mlp.experts.86.gate_proj", "model.layers.80.mlp.experts.87.gate_proj", "model.layers.80.mlp.experts.88.gate_proj", "model.layers.80.mlp.experts.89.gate_proj", "model.layers.80.mlp.experts.90.gate_proj", "model.layers.80.mlp.experts.91.gate_proj", "model.layers.80.mlp.experts.92.gate_proj", "model.layers.80.mlp.experts.93.gate_proj", "model.layers.80.mlp.experts.94.gate_proj", "model.layers.80.mlp.experts.95.gate_proj", "model.layers.80.mlp.experts.96.gate_proj", "model.layers.80.mlp.experts.97.gate_proj", "model.layers.80.mlp.experts.98.gate_proj", "model.layers.80.mlp.experts.99.gate_proj", "model.layers.80.mlp.experts.100.gate_proj", "model.layers.80.mlp.experts.101.gate_proj", "model.layers.80.mlp.experts.102.gate_proj", "model.layers.80.mlp.experts.103.gate_proj", "model.layers.80.mlp.experts.104.gate_proj", "model.layers.80.mlp.experts.105.gate_proj", "model.layers.80.mlp.experts.106.gate_proj", "model.layers.80.mlp.experts.107.gate_proj", "model.layers.80.mlp.experts.108.gate_proj", "model.layers.80.mlp.experts.109.gate_proj", "model.layers.80.mlp.experts.110.gate_proj", "model.layers.80.mlp.experts.111.gate_proj", "model.layers.80.mlp.experts.112.gate_proj", "model.layers.80.mlp.experts.113.gate_proj", "model.layers.80.mlp.experts.114.gate_proj", "model.layers.80.mlp.experts.115.gate_proj", "model.layers.80.mlp.experts.116.gate_proj", "model.layers.80.mlp.experts.117.gate_proj", "model.layers.80.mlp.experts.118.gate_proj", "model.layers.80.mlp.experts.119.gate_proj", "model.layers.80.mlp.experts.120.gate_proj", "model.layers.80.mlp.experts.121.gate_proj", "model.layers.80.mlp.experts.122.gate_proj", "model.layers.80.mlp.experts.123.gate_proj", "model.layers.80.mlp.experts.124.gate_proj", "model.layers.80.mlp.experts.125.gate_proj", "model.layers.80.mlp.experts.126.gate_proj", "model.layers.80.mlp.experts.127.gate_proj", "model.layers.80.mlp.experts.128.gate_proj", "model.layers.80.mlp.experts.129.gate_proj", "model.layers.80.mlp.experts.130.gate_proj", "model.layers.80.mlp.experts.131.gate_proj", "model.layers.80.mlp.experts.132.gate_proj", "model.layers.80.mlp.experts.133.gate_proj", "model.layers.80.mlp.experts.134.gate_proj", "model.layers.80.mlp.experts.135.gate_proj", "model.layers.80.mlp.experts.136.gate_proj", "model.layers.80.mlp.experts.137.gate_proj", "model.layers.80.mlp.experts.138.gate_proj", "model.layers.80.mlp.experts.139.gate_proj", "model.layers.80.mlp.experts.140.gate_proj", "model.layers.80.mlp.experts.141.gate_proj", "model.layers.80.mlp.experts.142.gate_proj", "model.layers.80.mlp.experts.143.gate_proj", "model.layers.80.mlp.experts.144.gate_proj", "model.layers.80.mlp.experts.145.gate_proj", "model.layers.80.mlp.experts.146.gate_proj", "model.layers.80.mlp.experts.147.gate_proj", "model.layers.80.mlp.experts.148.gate_proj", "model.layers.80.mlp.experts.149.gate_proj", "model.layers.80.mlp.experts.150.gate_proj", "model.layers.80.mlp.experts.151.gate_proj", "model.layers.80.mlp.experts.152.gate_proj", "model.layers.80.mlp.experts.153.gate_proj", "model.layers.80.mlp.experts.154.gate_proj", "model.layers.80.mlp.experts.155.gate_proj", "model.layers.80.mlp.experts.156.gate_proj", "model.layers.80.mlp.experts.157.gate_proj", "model.layers.80.mlp.experts.158.gate_proj", "model.layers.80.mlp.experts.159.gate_proj", "model.layers.80.mlp.experts.0.up_proj", "model.layers.80.mlp.experts.1.up_proj", "model.layers.80.mlp.experts.2.up_proj", "model.layers.80.mlp.experts.3.up_proj", "model.layers.80.mlp.experts.4.up_proj", "model.layers.80.mlp.experts.5.up_proj", "model.layers.80.mlp.experts.6.up_proj", "model.layers.80.mlp.experts.7.up_proj", "model.layers.80.mlp.experts.8.up_proj", "model.layers.80.mlp.experts.9.up_proj", "model.layers.80.mlp.experts.10.up_proj", "model.layers.80.mlp.experts.11.up_proj", "model.layers.80.mlp.experts.12.up_proj", "model.layers.80.mlp.experts.13.up_proj", "model.layers.80.mlp.experts.14.up_proj", "model.layers.80.mlp.experts.15.up_proj", "model.layers.80.mlp.experts.16.up_proj", "model.layers.80.mlp.experts.17.up_proj", "model.layers.80.mlp.experts.18.up_proj", "model.layers.80.mlp.experts.19.up_proj", "model.layers.80.mlp.experts.20.up_proj", "model.layers.80.mlp.experts.21.up_proj", "model.layers.80.mlp.experts.22.up_proj", "model.layers.80.mlp.experts.23.up_proj", "model.layers.80.mlp.experts.24.up_proj", "model.layers.80.mlp.experts.25.up_proj", "model.layers.80.mlp.experts.26.up_proj", "model.layers.80.mlp.experts.27.up_proj", "model.layers.80.mlp.experts.28.up_proj", "model.layers.80.mlp.experts.29.up_proj", "model.layers.80.mlp.experts.30.up_proj", "model.layers.80.mlp.experts.31.up_proj", "model.layers.80.mlp.experts.32.up_proj", "model.layers.80.mlp.experts.33.up_proj", "model.layers.80.mlp.experts.34.up_proj", "model.layers.80.mlp.experts.35.up_proj", "model.layers.80.mlp.experts.36.up_proj", "model.layers.80.mlp.experts.37.up_proj", "model.layers.80.mlp.experts.38.up_proj", "model.layers.80.mlp.experts.39.up_proj", "model.layers.80.mlp.experts.40.up_proj", "model.layers.80.mlp.experts.41.up_proj", "model.layers.80.mlp.experts.42.up_proj", "model.layers.80.mlp.experts.43.up_proj", "model.layers.80.mlp.experts.44.up_proj", "model.layers.80.mlp.experts.45.up_proj", "model.layers.80.mlp.experts.46.up_proj", "model.layers.80.mlp.experts.47.up_proj", "model.layers.80.mlp.experts.48.up_proj", "model.layers.80.mlp.experts.49.up_proj", "model.layers.80.mlp.experts.50.up_proj", "model.layers.80.mlp.experts.51.up_proj", "model.layers.80.mlp.experts.52.up_proj", "model.layers.80.mlp.experts.53.up_proj", "model.layers.80.mlp.experts.54.up_proj", "model.layers.80.mlp.experts.55.up_proj", "model.layers.80.mlp.experts.56.up_proj", "model.layers.80.mlp.experts.57.up_proj", "model.layers.80.mlp.experts.58.up_proj", "model.layers.80.mlp.experts.59.up_proj", "model.layers.80.mlp.experts.60.up_proj", "model.layers.80.mlp.experts.61.up_proj", "model.layers.80.mlp.experts.62.up_proj", "model.layers.80.mlp.experts.63.up_proj", "model.layers.80.mlp.experts.64.up_proj", "model.layers.80.mlp.experts.65.up_proj", "model.layers.80.mlp.experts.66.up_proj", "model.layers.80.mlp.experts.67.up_proj", "model.layers.80.mlp.experts.68.up_proj", "model.layers.80.mlp.experts.69.up_proj", "model.layers.80.mlp.experts.70.up_proj", "model.layers.80.mlp.experts.71.up_proj", "model.layers.80.mlp.experts.72.up_proj", "model.layers.80.mlp.experts.73.up_proj", "model.layers.80.mlp.experts.74.up_proj", "model.layers.80.mlp.experts.75.up_proj", "model.layers.80.mlp.experts.76.up_proj", "model.layers.80.mlp.experts.77.up_proj", "model.layers.80.mlp.experts.78.up_proj", "model.layers.80.mlp.experts.79.up_proj", "model.layers.80.mlp.experts.80.up_proj", "model.layers.80.mlp.experts.81.up_proj", "model.layers.80.mlp.experts.82.up_proj", "model.layers.80.mlp.experts.83.up_proj", "model.layers.80.mlp.experts.84.up_proj", "model.layers.80.mlp.experts.85.up_proj", "model.layers.80.mlp.experts.86.up_proj", "model.layers.80.mlp.experts.87.up_proj", "model.layers.80.mlp.experts.88.up_proj", "model.layers.80.mlp.experts.89.up_proj", "model.layers.80.mlp.experts.90.up_proj", "model.layers.80.mlp.experts.91.up_proj", "model.layers.80.mlp.experts.92.up_proj", "model.layers.80.mlp.experts.93.up_proj", "model.layers.80.mlp.experts.94.up_proj", "model.layers.80.mlp.experts.95.up_proj", "model.layers.80.mlp.experts.96.up_proj", "model.layers.80.mlp.experts.97.up_proj", "model.layers.80.mlp.experts.98.up_proj", "model.layers.80.mlp.experts.99.up_proj", "model.layers.80.mlp.experts.100.up_proj", "model.layers.80.mlp.experts.101.up_proj", "model.layers.80.mlp.experts.102.up_proj", "model.layers.80.mlp.experts.103.up_proj", "model.layers.80.mlp.experts.104.up_proj", "model.layers.80.mlp.experts.105.up_proj", "model.layers.80.mlp.experts.106.up_proj", "model.layers.80.mlp.experts.107.up_proj", "model.layers.80.mlp.experts.108.up_proj", "model.layers.80.mlp.experts.109.up_proj", "model.layers.80.mlp.experts.110.up_proj", "model.layers.80.mlp.experts.111.up_proj", "model.layers.80.mlp.experts.112.up_proj", "model.layers.80.mlp.experts.113.up_proj", "model.layers.80.mlp.experts.114.up_proj", "model.layers.80.mlp.experts.115.up_proj", "model.layers.80.mlp.experts.116.up_proj", "model.layers.80.mlp.experts.117.up_proj", "model.layers.80.mlp.experts.118.up_proj", "model.layers.80.mlp.experts.119.up_proj", "model.layers.80.mlp.experts.120.up_proj", "model.layers.80.mlp.experts.121.up_proj", "model.layers.80.mlp.experts.122.up_proj", "model.layers.80.mlp.experts.123.up_proj", "model.layers.80.mlp.experts.124.up_proj", "model.layers.80.mlp.experts.125.up_proj", "model.layers.80.mlp.experts.126.up_proj", "model.layers.80.mlp.experts.127.up_proj", "model.layers.80.mlp.experts.128.up_proj", "model.layers.80.mlp.experts.129.up_proj", "model.layers.80.mlp.experts.130.up_proj", "model.layers.80.mlp.experts.131.up_proj", "model.layers.80.mlp.experts.132.up_proj", "model.layers.80.mlp.experts.133.up_proj", "model.layers.80.mlp.experts.134.up_proj", "model.layers.80.mlp.experts.135.up_proj", "model.layers.80.mlp.experts.136.up_proj", "model.layers.80.mlp.experts.137.up_proj", "model.layers.80.mlp.experts.138.up_proj", "model.layers.80.mlp.experts.139.up_proj", "model.layers.80.mlp.experts.140.up_proj", "model.layers.80.mlp.experts.141.up_proj", "model.layers.80.mlp.experts.142.up_proj", "model.layers.80.mlp.experts.143.up_proj", "model.layers.80.mlp.experts.144.up_proj", "model.layers.80.mlp.experts.145.up_proj", "model.layers.80.mlp.experts.146.up_proj", "model.layers.80.mlp.experts.147.up_proj", "model.layers.80.mlp.experts.148.up_proj", "model.layers.80.mlp.experts.149.up_proj", "model.layers.80.mlp.experts.150.up_proj", "model.layers.80.mlp.experts.151.up_proj", "model.layers.80.mlp.experts.152.up_proj", "model.layers.80.mlp.experts.153.up_proj", "model.layers.80.mlp.experts.154.up_proj", "model.layers.80.mlp.experts.155.up_proj", "model.layers.80.mlp.experts.156.up_proj", "model.layers.80.mlp.experts.157.up_proj", "model.layers.80.mlp.experts.158.up_proj", "model.layers.80.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.9383430955931327e-06, "dbits": 5033164800 } ] }, { "idx": 482, "layers": [ "model.layers.80.mlp.experts.0.down_proj", "model.layers.80.mlp.experts.1.down_proj", "model.layers.80.mlp.experts.2.down_proj", "model.layers.80.mlp.experts.3.down_proj", "model.layers.80.mlp.experts.4.down_proj", "model.layers.80.mlp.experts.5.down_proj", "model.layers.80.mlp.experts.6.down_proj", "model.layers.80.mlp.experts.7.down_proj", "model.layers.80.mlp.experts.8.down_proj", "model.layers.80.mlp.experts.9.down_proj", "model.layers.80.mlp.experts.10.down_proj", "model.layers.80.mlp.experts.11.down_proj", "model.layers.80.mlp.experts.12.down_proj", "model.layers.80.mlp.experts.13.down_proj", "model.layers.80.mlp.experts.14.down_proj", "model.layers.80.mlp.experts.15.down_proj", "model.layers.80.mlp.experts.16.down_proj", "model.layers.80.mlp.experts.17.down_proj", "model.layers.80.mlp.experts.18.down_proj", "model.layers.80.mlp.experts.19.down_proj", "model.layers.80.mlp.experts.20.down_proj", "model.layers.80.mlp.experts.21.down_proj", "model.layers.80.mlp.experts.22.down_proj", "model.layers.80.mlp.experts.23.down_proj", "model.layers.80.mlp.experts.24.down_proj", "model.layers.80.mlp.experts.25.down_proj", "model.layers.80.mlp.experts.26.down_proj", "model.layers.80.mlp.experts.27.down_proj", "model.layers.80.mlp.experts.28.down_proj", "model.layers.80.mlp.experts.29.down_proj", "model.layers.80.mlp.experts.30.down_proj", "model.layers.80.mlp.experts.31.down_proj", "model.layers.80.mlp.experts.32.down_proj", "model.layers.80.mlp.experts.33.down_proj", "model.layers.80.mlp.experts.34.down_proj", "model.layers.80.mlp.experts.35.down_proj", "model.layers.80.mlp.experts.36.down_proj", "model.layers.80.mlp.experts.37.down_proj", "model.layers.80.mlp.experts.38.down_proj", "model.layers.80.mlp.experts.39.down_proj", "model.layers.80.mlp.experts.40.down_proj", "model.layers.80.mlp.experts.41.down_proj", "model.layers.80.mlp.experts.42.down_proj", "model.layers.80.mlp.experts.43.down_proj", "model.layers.80.mlp.experts.44.down_proj", "model.layers.80.mlp.experts.45.down_proj", "model.layers.80.mlp.experts.46.down_proj", "model.layers.80.mlp.experts.47.down_proj", "model.layers.80.mlp.experts.48.down_proj", "model.layers.80.mlp.experts.49.down_proj", "model.layers.80.mlp.experts.50.down_proj", "model.layers.80.mlp.experts.51.down_proj", "model.layers.80.mlp.experts.52.down_proj", "model.layers.80.mlp.experts.53.down_proj", "model.layers.80.mlp.experts.54.down_proj", "model.layers.80.mlp.experts.55.down_proj", "model.layers.80.mlp.experts.56.down_proj", "model.layers.80.mlp.experts.57.down_proj", "model.layers.80.mlp.experts.58.down_proj", "model.layers.80.mlp.experts.59.down_proj", "model.layers.80.mlp.experts.60.down_proj", "model.layers.80.mlp.experts.61.down_proj", "model.layers.80.mlp.experts.62.down_proj", "model.layers.80.mlp.experts.63.down_proj", "model.layers.80.mlp.experts.64.down_proj", "model.layers.80.mlp.experts.65.down_proj", "model.layers.80.mlp.experts.66.down_proj", "model.layers.80.mlp.experts.67.down_proj", "model.layers.80.mlp.experts.68.down_proj", "model.layers.80.mlp.experts.69.down_proj", "model.layers.80.mlp.experts.70.down_proj", "model.layers.80.mlp.experts.71.down_proj", "model.layers.80.mlp.experts.72.down_proj", "model.layers.80.mlp.experts.73.down_proj", "model.layers.80.mlp.experts.74.down_proj", "model.layers.80.mlp.experts.75.down_proj", "model.layers.80.mlp.experts.76.down_proj", "model.layers.80.mlp.experts.77.down_proj", "model.layers.80.mlp.experts.78.down_proj", "model.layers.80.mlp.experts.79.down_proj", "model.layers.80.mlp.experts.80.down_proj", "model.layers.80.mlp.experts.81.down_proj", "model.layers.80.mlp.experts.82.down_proj", "model.layers.80.mlp.experts.83.down_proj", "model.layers.80.mlp.experts.84.down_proj", "model.layers.80.mlp.experts.85.down_proj", "model.layers.80.mlp.experts.86.down_proj", "model.layers.80.mlp.experts.87.down_proj", "model.layers.80.mlp.experts.88.down_proj", "model.layers.80.mlp.experts.89.down_proj", "model.layers.80.mlp.experts.90.down_proj", "model.layers.80.mlp.experts.91.down_proj", "model.layers.80.mlp.experts.92.down_proj", "model.layers.80.mlp.experts.93.down_proj", "model.layers.80.mlp.experts.94.down_proj", "model.layers.80.mlp.experts.95.down_proj", "model.layers.80.mlp.experts.96.down_proj", "model.layers.80.mlp.experts.97.down_proj", "model.layers.80.mlp.experts.98.down_proj", "model.layers.80.mlp.experts.99.down_proj", "model.layers.80.mlp.experts.100.down_proj", "model.layers.80.mlp.experts.101.down_proj", "model.layers.80.mlp.experts.102.down_proj", "model.layers.80.mlp.experts.103.down_proj", "model.layers.80.mlp.experts.104.down_proj", "model.layers.80.mlp.experts.105.down_proj", "model.layers.80.mlp.experts.106.down_proj", "model.layers.80.mlp.experts.107.down_proj", "model.layers.80.mlp.experts.108.down_proj", "model.layers.80.mlp.experts.109.down_proj", "model.layers.80.mlp.experts.110.down_proj", "model.layers.80.mlp.experts.111.down_proj", "model.layers.80.mlp.experts.112.down_proj", "model.layers.80.mlp.experts.113.down_proj", "model.layers.80.mlp.experts.114.down_proj", "model.layers.80.mlp.experts.115.down_proj", "model.layers.80.mlp.experts.116.down_proj", "model.layers.80.mlp.experts.117.down_proj", "model.layers.80.mlp.experts.118.down_proj", "model.layers.80.mlp.experts.119.down_proj", "model.layers.80.mlp.experts.120.down_proj", "model.layers.80.mlp.experts.121.down_proj", "model.layers.80.mlp.experts.122.down_proj", "model.layers.80.mlp.experts.123.down_proj", "model.layers.80.mlp.experts.124.down_proj", "model.layers.80.mlp.experts.125.down_proj", "model.layers.80.mlp.experts.126.down_proj", "model.layers.80.mlp.experts.127.down_proj", "model.layers.80.mlp.experts.128.down_proj", "model.layers.80.mlp.experts.129.down_proj", "model.layers.80.mlp.experts.130.down_proj", "model.layers.80.mlp.experts.131.down_proj", "model.layers.80.mlp.experts.132.down_proj", "model.layers.80.mlp.experts.133.down_proj", "model.layers.80.mlp.experts.134.down_proj", "model.layers.80.mlp.experts.135.down_proj", "model.layers.80.mlp.experts.136.down_proj", "model.layers.80.mlp.experts.137.down_proj", "model.layers.80.mlp.experts.138.down_proj", "model.layers.80.mlp.experts.139.down_proj", "model.layers.80.mlp.experts.140.down_proj", "model.layers.80.mlp.experts.141.down_proj", "model.layers.80.mlp.experts.142.down_proj", "model.layers.80.mlp.experts.143.down_proj", "model.layers.80.mlp.experts.144.down_proj", "model.layers.80.mlp.experts.145.down_proj", "model.layers.80.mlp.experts.146.down_proj", "model.layers.80.mlp.experts.147.down_proj", "model.layers.80.mlp.experts.148.down_proj", "model.layers.80.mlp.experts.149.down_proj", "model.layers.80.mlp.experts.150.down_proj", "model.layers.80.mlp.experts.151.down_proj", "model.layers.80.mlp.experts.152.down_proj", "model.layers.80.mlp.experts.153.down_proj", "model.layers.80.mlp.experts.154.down_proj", "model.layers.80.mlp.experts.155.down_proj", "model.layers.80.mlp.experts.156.down_proj", "model.layers.80.mlp.experts.157.down_proj", "model.layers.80.mlp.experts.158.down_proj", "model.layers.80.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.2209249436854882e-06, "dbits": 2516582400 } ] }, { "idx": 483, "layers": [ "model.layers.81.self_attn.q_proj" ], "candidates": [ { "dkld": 4.1518069338053465e-07, "dbits": 125829120 } ] }, { "idx": 484, "layers": [ "model.layers.81.self_attn.k_proj", "model.layers.81.self_attn.v_proj" ], "candidates": [ { "dkld": -3.885079058818541e-06, "dbits": 20971520 } ] }, { "idx": 485, "layers": [ "model.layers.81.self_attn.o_proj" ], "candidates": [ { "dkld": 3.7708407035097914e-06, "dbits": 125829120 } ] }, { "idx": 486, "layers": [ "model.layers.81.mlp.shared_experts.gate_proj", "model.layers.81.mlp.shared_experts.up_proj", "model.layers.81.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.3110554795710064e-07, "dbits": 47185920 } ] }, { "idx": 487, "layers": [ "model.layers.81.mlp.experts.0.gate_proj", "model.layers.81.mlp.experts.1.gate_proj", "model.layers.81.mlp.experts.2.gate_proj", "model.layers.81.mlp.experts.3.gate_proj", "model.layers.81.mlp.experts.4.gate_proj", "model.layers.81.mlp.experts.5.gate_proj", "model.layers.81.mlp.experts.6.gate_proj", "model.layers.81.mlp.experts.7.gate_proj", "model.layers.81.mlp.experts.8.gate_proj", "model.layers.81.mlp.experts.9.gate_proj", "model.layers.81.mlp.experts.10.gate_proj", "model.layers.81.mlp.experts.11.gate_proj", "model.layers.81.mlp.experts.12.gate_proj", "model.layers.81.mlp.experts.13.gate_proj", "model.layers.81.mlp.experts.14.gate_proj", "model.layers.81.mlp.experts.15.gate_proj", "model.layers.81.mlp.experts.16.gate_proj", "model.layers.81.mlp.experts.17.gate_proj", "model.layers.81.mlp.experts.18.gate_proj", "model.layers.81.mlp.experts.19.gate_proj", "model.layers.81.mlp.experts.20.gate_proj", "model.layers.81.mlp.experts.21.gate_proj", "model.layers.81.mlp.experts.22.gate_proj", "model.layers.81.mlp.experts.23.gate_proj", "model.layers.81.mlp.experts.24.gate_proj", "model.layers.81.mlp.experts.25.gate_proj", "model.layers.81.mlp.experts.26.gate_proj", "model.layers.81.mlp.experts.27.gate_proj", "model.layers.81.mlp.experts.28.gate_proj", "model.layers.81.mlp.experts.29.gate_proj", "model.layers.81.mlp.experts.30.gate_proj", "model.layers.81.mlp.experts.31.gate_proj", "model.layers.81.mlp.experts.32.gate_proj", "model.layers.81.mlp.experts.33.gate_proj", "model.layers.81.mlp.experts.34.gate_proj", "model.layers.81.mlp.experts.35.gate_proj", "model.layers.81.mlp.experts.36.gate_proj", "model.layers.81.mlp.experts.37.gate_proj", "model.layers.81.mlp.experts.38.gate_proj", "model.layers.81.mlp.experts.39.gate_proj", "model.layers.81.mlp.experts.40.gate_proj", "model.layers.81.mlp.experts.41.gate_proj", "model.layers.81.mlp.experts.42.gate_proj", "model.layers.81.mlp.experts.43.gate_proj", "model.layers.81.mlp.experts.44.gate_proj", "model.layers.81.mlp.experts.45.gate_proj", "model.layers.81.mlp.experts.46.gate_proj", "model.layers.81.mlp.experts.47.gate_proj", "model.layers.81.mlp.experts.48.gate_proj", "model.layers.81.mlp.experts.49.gate_proj", "model.layers.81.mlp.experts.50.gate_proj", "model.layers.81.mlp.experts.51.gate_proj", "model.layers.81.mlp.experts.52.gate_proj", "model.layers.81.mlp.experts.53.gate_proj", "model.layers.81.mlp.experts.54.gate_proj", "model.layers.81.mlp.experts.55.gate_proj", "model.layers.81.mlp.experts.56.gate_proj", "model.layers.81.mlp.experts.57.gate_proj", "model.layers.81.mlp.experts.58.gate_proj", "model.layers.81.mlp.experts.59.gate_proj", "model.layers.81.mlp.experts.60.gate_proj", "model.layers.81.mlp.experts.61.gate_proj", "model.layers.81.mlp.experts.62.gate_proj", "model.layers.81.mlp.experts.63.gate_proj", "model.layers.81.mlp.experts.64.gate_proj", "model.layers.81.mlp.experts.65.gate_proj", "model.layers.81.mlp.experts.66.gate_proj", "model.layers.81.mlp.experts.67.gate_proj", "model.layers.81.mlp.experts.68.gate_proj", "model.layers.81.mlp.experts.69.gate_proj", "model.layers.81.mlp.experts.70.gate_proj", "model.layers.81.mlp.experts.71.gate_proj", "model.layers.81.mlp.experts.72.gate_proj", "model.layers.81.mlp.experts.73.gate_proj", "model.layers.81.mlp.experts.74.gate_proj", "model.layers.81.mlp.experts.75.gate_proj", "model.layers.81.mlp.experts.76.gate_proj", "model.layers.81.mlp.experts.77.gate_proj", "model.layers.81.mlp.experts.78.gate_proj", "model.layers.81.mlp.experts.79.gate_proj", "model.layers.81.mlp.experts.80.gate_proj", "model.layers.81.mlp.experts.81.gate_proj", "model.layers.81.mlp.experts.82.gate_proj", "model.layers.81.mlp.experts.83.gate_proj", "model.layers.81.mlp.experts.84.gate_proj", "model.layers.81.mlp.experts.85.gate_proj", "model.layers.81.mlp.experts.86.gate_proj", "model.layers.81.mlp.experts.87.gate_proj", "model.layers.81.mlp.experts.88.gate_proj", "model.layers.81.mlp.experts.89.gate_proj", "model.layers.81.mlp.experts.90.gate_proj", "model.layers.81.mlp.experts.91.gate_proj", "model.layers.81.mlp.experts.92.gate_proj", "model.layers.81.mlp.experts.93.gate_proj", "model.layers.81.mlp.experts.94.gate_proj", "model.layers.81.mlp.experts.95.gate_proj", "model.layers.81.mlp.experts.96.gate_proj", "model.layers.81.mlp.experts.97.gate_proj", "model.layers.81.mlp.experts.98.gate_proj", "model.layers.81.mlp.experts.99.gate_proj", "model.layers.81.mlp.experts.100.gate_proj", "model.layers.81.mlp.experts.101.gate_proj", "model.layers.81.mlp.experts.102.gate_proj", "model.layers.81.mlp.experts.103.gate_proj", "model.layers.81.mlp.experts.104.gate_proj", "model.layers.81.mlp.experts.105.gate_proj", "model.layers.81.mlp.experts.106.gate_proj", "model.layers.81.mlp.experts.107.gate_proj", "model.layers.81.mlp.experts.108.gate_proj", "model.layers.81.mlp.experts.109.gate_proj", "model.layers.81.mlp.experts.110.gate_proj", "model.layers.81.mlp.experts.111.gate_proj", "model.layers.81.mlp.experts.112.gate_proj", "model.layers.81.mlp.experts.113.gate_proj", "model.layers.81.mlp.experts.114.gate_proj", "model.layers.81.mlp.experts.115.gate_proj", "model.layers.81.mlp.experts.116.gate_proj", "model.layers.81.mlp.experts.117.gate_proj", "model.layers.81.mlp.experts.118.gate_proj", "model.layers.81.mlp.experts.119.gate_proj", "model.layers.81.mlp.experts.120.gate_proj", "model.layers.81.mlp.experts.121.gate_proj", "model.layers.81.mlp.experts.122.gate_proj", "model.layers.81.mlp.experts.123.gate_proj", "model.layers.81.mlp.experts.124.gate_proj", "model.layers.81.mlp.experts.125.gate_proj", "model.layers.81.mlp.experts.126.gate_proj", "model.layers.81.mlp.experts.127.gate_proj", "model.layers.81.mlp.experts.128.gate_proj", "model.layers.81.mlp.experts.129.gate_proj", "model.layers.81.mlp.experts.130.gate_proj", "model.layers.81.mlp.experts.131.gate_proj", "model.layers.81.mlp.experts.132.gate_proj", "model.layers.81.mlp.experts.133.gate_proj", "model.layers.81.mlp.experts.134.gate_proj", "model.layers.81.mlp.experts.135.gate_proj", "model.layers.81.mlp.experts.136.gate_proj", "model.layers.81.mlp.experts.137.gate_proj", "model.layers.81.mlp.experts.138.gate_proj", "model.layers.81.mlp.experts.139.gate_proj", "model.layers.81.mlp.experts.140.gate_proj", "model.layers.81.mlp.experts.141.gate_proj", "model.layers.81.mlp.experts.142.gate_proj", "model.layers.81.mlp.experts.143.gate_proj", "model.layers.81.mlp.experts.144.gate_proj", "model.layers.81.mlp.experts.145.gate_proj", "model.layers.81.mlp.experts.146.gate_proj", "model.layers.81.mlp.experts.147.gate_proj", "model.layers.81.mlp.experts.148.gate_proj", "model.layers.81.mlp.experts.149.gate_proj", "model.layers.81.mlp.experts.150.gate_proj", "model.layers.81.mlp.experts.151.gate_proj", "model.layers.81.mlp.experts.152.gate_proj", "model.layers.81.mlp.experts.153.gate_proj", "model.layers.81.mlp.experts.154.gate_proj", "model.layers.81.mlp.experts.155.gate_proj", "model.layers.81.mlp.experts.156.gate_proj", "model.layers.81.mlp.experts.157.gate_proj", "model.layers.81.mlp.experts.158.gate_proj", "model.layers.81.mlp.experts.159.gate_proj", "model.layers.81.mlp.experts.0.up_proj", "model.layers.81.mlp.experts.1.up_proj", "model.layers.81.mlp.experts.2.up_proj", "model.layers.81.mlp.experts.3.up_proj", "model.layers.81.mlp.experts.4.up_proj", "model.layers.81.mlp.experts.5.up_proj", "model.layers.81.mlp.experts.6.up_proj", "model.layers.81.mlp.experts.7.up_proj", "model.layers.81.mlp.experts.8.up_proj", "model.layers.81.mlp.experts.9.up_proj", "model.layers.81.mlp.experts.10.up_proj", "model.layers.81.mlp.experts.11.up_proj", "model.layers.81.mlp.experts.12.up_proj", "model.layers.81.mlp.experts.13.up_proj", "model.layers.81.mlp.experts.14.up_proj", "model.layers.81.mlp.experts.15.up_proj", "model.layers.81.mlp.experts.16.up_proj", "model.layers.81.mlp.experts.17.up_proj", "model.layers.81.mlp.experts.18.up_proj", "model.layers.81.mlp.experts.19.up_proj", "model.layers.81.mlp.experts.20.up_proj", "model.layers.81.mlp.experts.21.up_proj", "model.layers.81.mlp.experts.22.up_proj", "model.layers.81.mlp.experts.23.up_proj", "model.layers.81.mlp.experts.24.up_proj", "model.layers.81.mlp.experts.25.up_proj", "model.layers.81.mlp.experts.26.up_proj", "model.layers.81.mlp.experts.27.up_proj", "model.layers.81.mlp.experts.28.up_proj", "model.layers.81.mlp.experts.29.up_proj", "model.layers.81.mlp.experts.30.up_proj", "model.layers.81.mlp.experts.31.up_proj", "model.layers.81.mlp.experts.32.up_proj", "model.layers.81.mlp.experts.33.up_proj", "model.layers.81.mlp.experts.34.up_proj", "model.layers.81.mlp.experts.35.up_proj", "model.layers.81.mlp.experts.36.up_proj", "model.layers.81.mlp.experts.37.up_proj", "model.layers.81.mlp.experts.38.up_proj", "model.layers.81.mlp.experts.39.up_proj", "model.layers.81.mlp.experts.40.up_proj", "model.layers.81.mlp.experts.41.up_proj", "model.layers.81.mlp.experts.42.up_proj", "model.layers.81.mlp.experts.43.up_proj", "model.layers.81.mlp.experts.44.up_proj", "model.layers.81.mlp.experts.45.up_proj", "model.layers.81.mlp.experts.46.up_proj", "model.layers.81.mlp.experts.47.up_proj", "model.layers.81.mlp.experts.48.up_proj", "model.layers.81.mlp.experts.49.up_proj", "model.layers.81.mlp.experts.50.up_proj", "model.layers.81.mlp.experts.51.up_proj", "model.layers.81.mlp.experts.52.up_proj", "model.layers.81.mlp.experts.53.up_proj", "model.layers.81.mlp.experts.54.up_proj", "model.layers.81.mlp.experts.55.up_proj", "model.layers.81.mlp.experts.56.up_proj", "model.layers.81.mlp.experts.57.up_proj", "model.layers.81.mlp.experts.58.up_proj", "model.layers.81.mlp.experts.59.up_proj", "model.layers.81.mlp.experts.60.up_proj", "model.layers.81.mlp.experts.61.up_proj", "model.layers.81.mlp.experts.62.up_proj", "model.layers.81.mlp.experts.63.up_proj", "model.layers.81.mlp.experts.64.up_proj", "model.layers.81.mlp.experts.65.up_proj", "model.layers.81.mlp.experts.66.up_proj", "model.layers.81.mlp.experts.67.up_proj", "model.layers.81.mlp.experts.68.up_proj", "model.layers.81.mlp.experts.69.up_proj", "model.layers.81.mlp.experts.70.up_proj", "model.layers.81.mlp.experts.71.up_proj", "model.layers.81.mlp.experts.72.up_proj", "model.layers.81.mlp.experts.73.up_proj", "model.layers.81.mlp.experts.74.up_proj", "model.layers.81.mlp.experts.75.up_proj", "model.layers.81.mlp.experts.76.up_proj", "model.layers.81.mlp.experts.77.up_proj", "model.layers.81.mlp.experts.78.up_proj", "model.layers.81.mlp.experts.79.up_proj", "model.layers.81.mlp.experts.80.up_proj", "model.layers.81.mlp.experts.81.up_proj", "model.layers.81.mlp.experts.82.up_proj", "model.layers.81.mlp.experts.83.up_proj", "model.layers.81.mlp.experts.84.up_proj", "model.layers.81.mlp.experts.85.up_proj", "model.layers.81.mlp.experts.86.up_proj", "model.layers.81.mlp.experts.87.up_proj", "model.layers.81.mlp.experts.88.up_proj", "model.layers.81.mlp.experts.89.up_proj", "model.layers.81.mlp.experts.90.up_proj", "model.layers.81.mlp.experts.91.up_proj", "model.layers.81.mlp.experts.92.up_proj", "model.layers.81.mlp.experts.93.up_proj", "model.layers.81.mlp.experts.94.up_proj", "model.layers.81.mlp.experts.95.up_proj", "model.layers.81.mlp.experts.96.up_proj", "model.layers.81.mlp.experts.97.up_proj", "model.layers.81.mlp.experts.98.up_proj", "model.layers.81.mlp.experts.99.up_proj", "model.layers.81.mlp.experts.100.up_proj", "model.layers.81.mlp.experts.101.up_proj", "model.layers.81.mlp.experts.102.up_proj", "model.layers.81.mlp.experts.103.up_proj", "model.layers.81.mlp.experts.104.up_proj", "model.layers.81.mlp.experts.105.up_proj", "model.layers.81.mlp.experts.106.up_proj", "model.layers.81.mlp.experts.107.up_proj", "model.layers.81.mlp.experts.108.up_proj", "model.layers.81.mlp.experts.109.up_proj", "model.layers.81.mlp.experts.110.up_proj", "model.layers.81.mlp.experts.111.up_proj", "model.layers.81.mlp.experts.112.up_proj", "model.layers.81.mlp.experts.113.up_proj", "model.layers.81.mlp.experts.114.up_proj", "model.layers.81.mlp.experts.115.up_proj", "model.layers.81.mlp.experts.116.up_proj", "model.layers.81.mlp.experts.117.up_proj", "model.layers.81.mlp.experts.118.up_proj", "model.layers.81.mlp.experts.119.up_proj", "model.layers.81.mlp.experts.120.up_proj", "model.layers.81.mlp.experts.121.up_proj", "model.layers.81.mlp.experts.122.up_proj", "model.layers.81.mlp.experts.123.up_proj", "model.layers.81.mlp.experts.124.up_proj", "model.layers.81.mlp.experts.125.up_proj", "model.layers.81.mlp.experts.126.up_proj", "model.layers.81.mlp.experts.127.up_proj", "model.layers.81.mlp.experts.128.up_proj", "model.layers.81.mlp.experts.129.up_proj", "model.layers.81.mlp.experts.130.up_proj", "model.layers.81.mlp.experts.131.up_proj", "model.layers.81.mlp.experts.132.up_proj", "model.layers.81.mlp.experts.133.up_proj", "model.layers.81.mlp.experts.134.up_proj", "model.layers.81.mlp.experts.135.up_proj", "model.layers.81.mlp.experts.136.up_proj", "model.layers.81.mlp.experts.137.up_proj", "model.layers.81.mlp.experts.138.up_proj", "model.layers.81.mlp.experts.139.up_proj", "model.layers.81.mlp.experts.140.up_proj", "model.layers.81.mlp.experts.141.up_proj", "model.layers.81.mlp.experts.142.up_proj", "model.layers.81.mlp.experts.143.up_proj", "model.layers.81.mlp.experts.144.up_proj", "model.layers.81.mlp.experts.145.up_proj", "model.layers.81.mlp.experts.146.up_proj", "model.layers.81.mlp.experts.147.up_proj", "model.layers.81.mlp.experts.148.up_proj", "model.layers.81.mlp.experts.149.up_proj", "model.layers.81.mlp.experts.150.up_proj", "model.layers.81.mlp.experts.151.up_proj", "model.layers.81.mlp.experts.152.up_proj", "model.layers.81.mlp.experts.153.up_proj", "model.layers.81.mlp.experts.154.up_proj", "model.layers.81.mlp.experts.155.up_proj", "model.layers.81.mlp.experts.156.up_proj", "model.layers.81.mlp.experts.157.up_proj", "model.layers.81.mlp.experts.158.up_proj", "model.layers.81.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.795368734747128e-06, "dbits": 5033164800 } ] }, { "idx": 488, "layers": [ "model.layers.81.mlp.experts.0.down_proj", "model.layers.81.mlp.experts.1.down_proj", "model.layers.81.mlp.experts.2.down_proj", "model.layers.81.mlp.experts.3.down_proj", "model.layers.81.mlp.experts.4.down_proj", "model.layers.81.mlp.experts.5.down_proj", "model.layers.81.mlp.experts.6.down_proj", "model.layers.81.mlp.experts.7.down_proj", "model.layers.81.mlp.experts.8.down_proj", "model.layers.81.mlp.experts.9.down_proj", "model.layers.81.mlp.experts.10.down_proj", "model.layers.81.mlp.experts.11.down_proj", "model.layers.81.mlp.experts.12.down_proj", "model.layers.81.mlp.experts.13.down_proj", "model.layers.81.mlp.experts.14.down_proj", "model.layers.81.mlp.experts.15.down_proj", "model.layers.81.mlp.experts.16.down_proj", "model.layers.81.mlp.experts.17.down_proj", "model.layers.81.mlp.experts.18.down_proj", "model.layers.81.mlp.experts.19.down_proj", "model.layers.81.mlp.experts.20.down_proj", "model.layers.81.mlp.experts.21.down_proj", "model.layers.81.mlp.experts.22.down_proj", "model.layers.81.mlp.experts.23.down_proj", "model.layers.81.mlp.experts.24.down_proj", "model.layers.81.mlp.experts.25.down_proj", "model.layers.81.mlp.experts.26.down_proj", "model.layers.81.mlp.experts.27.down_proj", "model.layers.81.mlp.experts.28.down_proj", "model.layers.81.mlp.experts.29.down_proj", "model.layers.81.mlp.experts.30.down_proj", "model.layers.81.mlp.experts.31.down_proj", "model.layers.81.mlp.experts.32.down_proj", "model.layers.81.mlp.experts.33.down_proj", "model.layers.81.mlp.experts.34.down_proj", "model.layers.81.mlp.experts.35.down_proj", "model.layers.81.mlp.experts.36.down_proj", "model.layers.81.mlp.experts.37.down_proj", "model.layers.81.mlp.experts.38.down_proj", "model.layers.81.mlp.experts.39.down_proj", "model.layers.81.mlp.experts.40.down_proj", "model.layers.81.mlp.experts.41.down_proj", "model.layers.81.mlp.experts.42.down_proj", "model.layers.81.mlp.experts.43.down_proj", "model.layers.81.mlp.experts.44.down_proj", "model.layers.81.mlp.experts.45.down_proj", "model.layers.81.mlp.experts.46.down_proj", "model.layers.81.mlp.experts.47.down_proj", "model.layers.81.mlp.experts.48.down_proj", "model.layers.81.mlp.experts.49.down_proj", "model.layers.81.mlp.experts.50.down_proj", "model.layers.81.mlp.experts.51.down_proj", "model.layers.81.mlp.experts.52.down_proj", "model.layers.81.mlp.experts.53.down_proj", "model.layers.81.mlp.experts.54.down_proj", "model.layers.81.mlp.experts.55.down_proj", "model.layers.81.mlp.experts.56.down_proj", "model.layers.81.mlp.experts.57.down_proj", "model.layers.81.mlp.experts.58.down_proj", "model.layers.81.mlp.experts.59.down_proj", "model.layers.81.mlp.experts.60.down_proj", "model.layers.81.mlp.experts.61.down_proj", "model.layers.81.mlp.experts.62.down_proj", "model.layers.81.mlp.experts.63.down_proj", "model.layers.81.mlp.experts.64.down_proj", "model.layers.81.mlp.experts.65.down_proj", "model.layers.81.mlp.experts.66.down_proj", "model.layers.81.mlp.experts.67.down_proj", "model.layers.81.mlp.experts.68.down_proj", "model.layers.81.mlp.experts.69.down_proj", "model.layers.81.mlp.experts.70.down_proj", "model.layers.81.mlp.experts.71.down_proj", "model.layers.81.mlp.experts.72.down_proj", "model.layers.81.mlp.experts.73.down_proj", "model.layers.81.mlp.experts.74.down_proj", "model.layers.81.mlp.experts.75.down_proj", "model.layers.81.mlp.experts.76.down_proj", "model.layers.81.mlp.experts.77.down_proj", "model.layers.81.mlp.experts.78.down_proj", "model.layers.81.mlp.experts.79.down_proj", "model.layers.81.mlp.experts.80.down_proj", "model.layers.81.mlp.experts.81.down_proj", "model.layers.81.mlp.experts.82.down_proj", "model.layers.81.mlp.experts.83.down_proj", "model.layers.81.mlp.experts.84.down_proj", "model.layers.81.mlp.experts.85.down_proj", "model.layers.81.mlp.experts.86.down_proj", "model.layers.81.mlp.experts.87.down_proj", "model.layers.81.mlp.experts.88.down_proj", "model.layers.81.mlp.experts.89.down_proj", "model.layers.81.mlp.experts.90.down_proj", "model.layers.81.mlp.experts.91.down_proj", "model.layers.81.mlp.experts.92.down_proj", "model.layers.81.mlp.experts.93.down_proj", "model.layers.81.mlp.experts.94.down_proj", "model.layers.81.mlp.experts.95.down_proj", "model.layers.81.mlp.experts.96.down_proj", "model.layers.81.mlp.experts.97.down_proj", "model.layers.81.mlp.experts.98.down_proj", "model.layers.81.mlp.experts.99.down_proj", "model.layers.81.mlp.experts.100.down_proj", "model.layers.81.mlp.experts.101.down_proj", "model.layers.81.mlp.experts.102.down_proj", "model.layers.81.mlp.experts.103.down_proj", "model.layers.81.mlp.experts.104.down_proj", "model.layers.81.mlp.experts.105.down_proj", "model.layers.81.mlp.experts.106.down_proj", "model.layers.81.mlp.experts.107.down_proj", "model.layers.81.mlp.experts.108.down_proj", "model.layers.81.mlp.experts.109.down_proj", "model.layers.81.mlp.experts.110.down_proj", "model.layers.81.mlp.experts.111.down_proj", "model.layers.81.mlp.experts.112.down_proj", "model.layers.81.mlp.experts.113.down_proj", "model.layers.81.mlp.experts.114.down_proj", "model.layers.81.mlp.experts.115.down_proj", "model.layers.81.mlp.experts.116.down_proj", "model.layers.81.mlp.experts.117.down_proj", "model.layers.81.mlp.experts.118.down_proj", "model.layers.81.mlp.experts.119.down_proj", "model.layers.81.mlp.experts.120.down_proj", "model.layers.81.mlp.experts.121.down_proj", "model.layers.81.mlp.experts.122.down_proj", "model.layers.81.mlp.experts.123.down_proj", "model.layers.81.mlp.experts.124.down_proj", "model.layers.81.mlp.experts.125.down_proj", "model.layers.81.mlp.experts.126.down_proj", "model.layers.81.mlp.experts.127.down_proj", "model.layers.81.mlp.experts.128.down_proj", "model.layers.81.mlp.experts.129.down_proj", "model.layers.81.mlp.experts.130.down_proj", "model.layers.81.mlp.experts.131.down_proj", "model.layers.81.mlp.experts.132.down_proj", "model.layers.81.mlp.experts.133.down_proj", "model.layers.81.mlp.experts.134.down_proj", "model.layers.81.mlp.experts.135.down_proj", "model.layers.81.mlp.experts.136.down_proj", "model.layers.81.mlp.experts.137.down_proj", "model.layers.81.mlp.experts.138.down_proj", "model.layers.81.mlp.experts.139.down_proj", "model.layers.81.mlp.experts.140.down_proj", "model.layers.81.mlp.experts.141.down_proj", "model.layers.81.mlp.experts.142.down_proj", "model.layers.81.mlp.experts.143.down_proj", "model.layers.81.mlp.experts.144.down_proj", "model.layers.81.mlp.experts.145.down_proj", "model.layers.81.mlp.experts.146.down_proj", "model.layers.81.mlp.experts.147.down_proj", "model.layers.81.mlp.experts.148.down_proj", "model.layers.81.mlp.experts.149.down_proj", "model.layers.81.mlp.experts.150.down_proj", "model.layers.81.mlp.experts.151.down_proj", "model.layers.81.mlp.experts.152.down_proj", "model.layers.81.mlp.experts.153.down_proj", "model.layers.81.mlp.experts.154.down_proj", "model.layers.81.mlp.experts.155.down_proj", "model.layers.81.mlp.experts.156.down_proj", "model.layers.81.mlp.experts.157.down_proj", "model.layers.81.mlp.experts.158.down_proj", "model.layers.81.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.7776957722380324e-06, "dbits": 2516582400 } ] }, { "idx": 489, "layers": [ "model.layers.82.self_attn.q_proj" ], "candidates": [ { "dkld": 1.8713995814323642e-06, "dbits": 125829120 } ] }, { "idx": 490, "layers": [ "model.layers.82.self_attn.k_proj", "model.layers.82.self_attn.v_proj" ], "candidates": [ { "dkld": -5.133461672812657e-06, "dbits": 20971520 } ] }, { "idx": 491, "layers": [ "model.layers.82.self_attn.o_proj" ], "candidates": [ { "dkld": -2.3277971195057034e-06, "dbits": 125829120 } ] }, { "idx": 492, "layers": [ "model.layers.82.mlp.shared_experts.gate_proj", "model.layers.82.mlp.shared_experts.up_proj", "model.layers.82.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.6727502699941397e-06, "dbits": 47185920 } ] }, { "idx": 493, "layers": [ "model.layers.82.mlp.experts.0.gate_proj", "model.layers.82.mlp.experts.1.gate_proj", "model.layers.82.mlp.experts.2.gate_proj", "model.layers.82.mlp.experts.3.gate_proj", "model.layers.82.mlp.experts.4.gate_proj", "model.layers.82.mlp.experts.5.gate_proj", "model.layers.82.mlp.experts.6.gate_proj", "model.layers.82.mlp.experts.7.gate_proj", "model.layers.82.mlp.experts.8.gate_proj", "model.layers.82.mlp.experts.9.gate_proj", "model.layers.82.mlp.experts.10.gate_proj", "model.layers.82.mlp.experts.11.gate_proj", "model.layers.82.mlp.experts.12.gate_proj", "model.layers.82.mlp.experts.13.gate_proj", "model.layers.82.mlp.experts.14.gate_proj", "model.layers.82.mlp.experts.15.gate_proj", "model.layers.82.mlp.experts.16.gate_proj", "model.layers.82.mlp.experts.17.gate_proj", "model.layers.82.mlp.experts.18.gate_proj", "model.layers.82.mlp.experts.19.gate_proj", "model.layers.82.mlp.experts.20.gate_proj", "model.layers.82.mlp.experts.21.gate_proj", "model.layers.82.mlp.experts.22.gate_proj", "model.layers.82.mlp.experts.23.gate_proj", "model.layers.82.mlp.experts.24.gate_proj", "model.layers.82.mlp.experts.25.gate_proj", "model.layers.82.mlp.experts.26.gate_proj", "model.layers.82.mlp.experts.27.gate_proj", "model.layers.82.mlp.experts.28.gate_proj", "model.layers.82.mlp.experts.29.gate_proj", "model.layers.82.mlp.experts.30.gate_proj", "model.layers.82.mlp.experts.31.gate_proj", "model.layers.82.mlp.experts.32.gate_proj", "model.layers.82.mlp.experts.33.gate_proj", "model.layers.82.mlp.experts.34.gate_proj", "model.layers.82.mlp.experts.35.gate_proj", "model.layers.82.mlp.experts.36.gate_proj", "model.layers.82.mlp.experts.37.gate_proj", "model.layers.82.mlp.experts.38.gate_proj", "model.layers.82.mlp.experts.39.gate_proj", "model.layers.82.mlp.experts.40.gate_proj", "model.layers.82.mlp.experts.41.gate_proj", "model.layers.82.mlp.experts.42.gate_proj", "model.layers.82.mlp.experts.43.gate_proj", "model.layers.82.mlp.experts.44.gate_proj", "model.layers.82.mlp.experts.45.gate_proj", "model.layers.82.mlp.experts.46.gate_proj", "model.layers.82.mlp.experts.47.gate_proj", "model.layers.82.mlp.experts.48.gate_proj", "model.layers.82.mlp.experts.49.gate_proj", "model.layers.82.mlp.experts.50.gate_proj", "model.layers.82.mlp.experts.51.gate_proj", "model.layers.82.mlp.experts.52.gate_proj", "model.layers.82.mlp.experts.53.gate_proj", "model.layers.82.mlp.experts.54.gate_proj", "model.layers.82.mlp.experts.55.gate_proj", "model.layers.82.mlp.experts.56.gate_proj", "model.layers.82.mlp.experts.57.gate_proj", "model.layers.82.mlp.experts.58.gate_proj", "model.layers.82.mlp.experts.59.gate_proj", "model.layers.82.mlp.experts.60.gate_proj", "model.layers.82.mlp.experts.61.gate_proj", "model.layers.82.mlp.experts.62.gate_proj", "model.layers.82.mlp.experts.63.gate_proj", "model.layers.82.mlp.experts.64.gate_proj", "model.layers.82.mlp.experts.65.gate_proj", "model.layers.82.mlp.experts.66.gate_proj", "model.layers.82.mlp.experts.67.gate_proj", "model.layers.82.mlp.experts.68.gate_proj", "model.layers.82.mlp.experts.69.gate_proj", "model.layers.82.mlp.experts.70.gate_proj", "model.layers.82.mlp.experts.71.gate_proj", "model.layers.82.mlp.experts.72.gate_proj", "model.layers.82.mlp.experts.73.gate_proj", "model.layers.82.mlp.experts.74.gate_proj", "model.layers.82.mlp.experts.75.gate_proj", "model.layers.82.mlp.experts.76.gate_proj", "model.layers.82.mlp.experts.77.gate_proj", "model.layers.82.mlp.experts.78.gate_proj", "model.layers.82.mlp.experts.79.gate_proj", "model.layers.82.mlp.experts.80.gate_proj", "model.layers.82.mlp.experts.81.gate_proj", "model.layers.82.mlp.experts.82.gate_proj", "model.layers.82.mlp.experts.83.gate_proj", "model.layers.82.mlp.experts.84.gate_proj", "model.layers.82.mlp.experts.85.gate_proj", "model.layers.82.mlp.experts.86.gate_proj", "model.layers.82.mlp.experts.87.gate_proj", "model.layers.82.mlp.experts.88.gate_proj", "model.layers.82.mlp.experts.89.gate_proj", "model.layers.82.mlp.experts.90.gate_proj", "model.layers.82.mlp.experts.91.gate_proj", "model.layers.82.mlp.experts.92.gate_proj", "model.layers.82.mlp.experts.93.gate_proj", "model.layers.82.mlp.experts.94.gate_proj", "model.layers.82.mlp.experts.95.gate_proj", "model.layers.82.mlp.experts.96.gate_proj", "model.layers.82.mlp.experts.97.gate_proj", "model.layers.82.mlp.experts.98.gate_proj", "model.layers.82.mlp.experts.99.gate_proj", "model.layers.82.mlp.experts.100.gate_proj", "model.layers.82.mlp.experts.101.gate_proj", "model.layers.82.mlp.experts.102.gate_proj", "model.layers.82.mlp.experts.103.gate_proj", "model.layers.82.mlp.experts.104.gate_proj", "model.layers.82.mlp.experts.105.gate_proj", "model.layers.82.mlp.experts.106.gate_proj", "model.layers.82.mlp.experts.107.gate_proj", "model.layers.82.mlp.experts.108.gate_proj", "model.layers.82.mlp.experts.109.gate_proj", "model.layers.82.mlp.experts.110.gate_proj", "model.layers.82.mlp.experts.111.gate_proj", "model.layers.82.mlp.experts.112.gate_proj", "model.layers.82.mlp.experts.113.gate_proj", "model.layers.82.mlp.experts.114.gate_proj", "model.layers.82.mlp.experts.115.gate_proj", "model.layers.82.mlp.experts.116.gate_proj", "model.layers.82.mlp.experts.117.gate_proj", "model.layers.82.mlp.experts.118.gate_proj", "model.layers.82.mlp.experts.119.gate_proj", "model.layers.82.mlp.experts.120.gate_proj", "model.layers.82.mlp.experts.121.gate_proj", "model.layers.82.mlp.experts.122.gate_proj", "model.layers.82.mlp.experts.123.gate_proj", "model.layers.82.mlp.experts.124.gate_proj", "model.layers.82.mlp.experts.125.gate_proj", "model.layers.82.mlp.experts.126.gate_proj", "model.layers.82.mlp.experts.127.gate_proj", "model.layers.82.mlp.experts.128.gate_proj", "model.layers.82.mlp.experts.129.gate_proj", "model.layers.82.mlp.experts.130.gate_proj", "model.layers.82.mlp.experts.131.gate_proj", "model.layers.82.mlp.experts.132.gate_proj", "model.layers.82.mlp.experts.133.gate_proj", "model.layers.82.mlp.experts.134.gate_proj", "model.layers.82.mlp.experts.135.gate_proj", "model.layers.82.mlp.experts.136.gate_proj", "model.layers.82.mlp.experts.137.gate_proj", "model.layers.82.mlp.experts.138.gate_proj", "model.layers.82.mlp.experts.139.gate_proj", "model.layers.82.mlp.experts.140.gate_proj", "model.layers.82.mlp.experts.141.gate_proj", "model.layers.82.mlp.experts.142.gate_proj", "model.layers.82.mlp.experts.143.gate_proj", "model.layers.82.mlp.experts.144.gate_proj", "model.layers.82.mlp.experts.145.gate_proj", "model.layers.82.mlp.experts.146.gate_proj", "model.layers.82.mlp.experts.147.gate_proj", "model.layers.82.mlp.experts.148.gate_proj", "model.layers.82.mlp.experts.149.gate_proj", "model.layers.82.mlp.experts.150.gate_proj", "model.layers.82.mlp.experts.151.gate_proj", "model.layers.82.mlp.experts.152.gate_proj", "model.layers.82.mlp.experts.153.gate_proj", "model.layers.82.mlp.experts.154.gate_proj", "model.layers.82.mlp.experts.155.gate_proj", "model.layers.82.mlp.experts.156.gate_proj", "model.layers.82.mlp.experts.157.gate_proj", "model.layers.82.mlp.experts.158.gate_proj", "model.layers.82.mlp.experts.159.gate_proj", "model.layers.82.mlp.experts.0.up_proj", "model.layers.82.mlp.experts.1.up_proj", "model.layers.82.mlp.experts.2.up_proj", "model.layers.82.mlp.experts.3.up_proj", "model.layers.82.mlp.experts.4.up_proj", "model.layers.82.mlp.experts.5.up_proj", "model.layers.82.mlp.experts.6.up_proj", "model.layers.82.mlp.experts.7.up_proj", "model.layers.82.mlp.experts.8.up_proj", "model.layers.82.mlp.experts.9.up_proj", "model.layers.82.mlp.experts.10.up_proj", "model.layers.82.mlp.experts.11.up_proj", "model.layers.82.mlp.experts.12.up_proj", "model.layers.82.mlp.experts.13.up_proj", "model.layers.82.mlp.experts.14.up_proj", "model.layers.82.mlp.experts.15.up_proj", "model.layers.82.mlp.experts.16.up_proj", "model.layers.82.mlp.experts.17.up_proj", "model.layers.82.mlp.experts.18.up_proj", "model.layers.82.mlp.experts.19.up_proj", "model.layers.82.mlp.experts.20.up_proj", "model.layers.82.mlp.experts.21.up_proj", "model.layers.82.mlp.experts.22.up_proj", "model.layers.82.mlp.experts.23.up_proj", "model.layers.82.mlp.experts.24.up_proj", "model.layers.82.mlp.experts.25.up_proj", "model.layers.82.mlp.experts.26.up_proj", "model.layers.82.mlp.experts.27.up_proj", "model.layers.82.mlp.experts.28.up_proj", "model.layers.82.mlp.experts.29.up_proj", "model.layers.82.mlp.experts.30.up_proj", "model.layers.82.mlp.experts.31.up_proj", "model.layers.82.mlp.experts.32.up_proj", "model.layers.82.mlp.experts.33.up_proj", "model.layers.82.mlp.experts.34.up_proj", "model.layers.82.mlp.experts.35.up_proj", "model.layers.82.mlp.experts.36.up_proj", "model.layers.82.mlp.experts.37.up_proj", "model.layers.82.mlp.experts.38.up_proj", "model.layers.82.mlp.experts.39.up_proj", "model.layers.82.mlp.experts.40.up_proj", "model.layers.82.mlp.experts.41.up_proj", "model.layers.82.mlp.experts.42.up_proj", "model.layers.82.mlp.experts.43.up_proj", "model.layers.82.mlp.experts.44.up_proj", "model.layers.82.mlp.experts.45.up_proj", "model.layers.82.mlp.experts.46.up_proj", "model.layers.82.mlp.experts.47.up_proj", "model.layers.82.mlp.experts.48.up_proj", "model.layers.82.mlp.experts.49.up_proj", "model.layers.82.mlp.experts.50.up_proj", "model.layers.82.mlp.experts.51.up_proj", "model.layers.82.mlp.experts.52.up_proj", "model.layers.82.mlp.experts.53.up_proj", "model.layers.82.mlp.experts.54.up_proj", "model.layers.82.mlp.experts.55.up_proj", "model.layers.82.mlp.experts.56.up_proj", "model.layers.82.mlp.experts.57.up_proj", "model.layers.82.mlp.experts.58.up_proj", "model.layers.82.mlp.experts.59.up_proj", "model.layers.82.mlp.experts.60.up_proj", "model.layers.82.mlp.experts.61.up_proj", "model.layers.82.mlp.experts.62.up_proj", "model.layers.82.mlp.experts.63.up_proj", "model.layers.82.mlp.experts.64.up_proj", "model.layers.82.mlp.experts.65.up_proj", "model.layers.82.mlp.experts.66.up_proj", "model.layers.82.mlp.experts.67.up_proj", "model.layers.82.mlp.experts.68.up_proj", "model.layers.82.mlp.experts.69.up_proj", "model.layers.82.mlp.experts.70.up_proj", "model.layers.82.mlp.experts.71.up_proj", "model.layers.82.mlp.experts.72.up_proj", "model.layers.82.mlp.experts.73.up_proj", "model.layers.82.mlp.experts.74.up_proj", "model.layers.82.mlp.experts.75.up_proj", "model.layers.82.mlp.experts.76.up_proj", "model.layers.82.mlp.experts.77.up_proj", "model.layers.82.mlp.experts.78.up_proj", "model.layers.82.mlp.experts.79.up_proj", "model.layers.82.mlp.experts.80.up_proj", "model.layers.82.mlp.experts.81.up_proj", "model.layers.82.mlp.experts.82.up_proj", "model.layers.82.mlp.experts.83.up_proj", "model.layers.82.mlp.experts.84.up_proj", "model.layers.82.mlp.experts.85.up_proj", "model.layers.82.mlp.experts.86.up_proj", "model.layers.82.mlp.experts.87.up_proj", "model.layers.82.mlp.experts.88.up_proj", "model.layers.82.mlp.experts.89.up_proj", "model.layers.82.mlp.experts.90.up_proj", "model.layers.82.mlp.experts.91.up_proj", "model.layers.82.mlp.experts.92.up_proj", "model.layers.82.mlp.experts.93.up_proj", "model.layers.82.mlp.experts.94.up_proj", "model.layers.82.mlp.experts.95.up_proj", "model.layers.82.mlp.experts.96.up_proj", "model.layers.82.mlp.experts.97.up_proj", "model.layers.82.mlp.experts.98.up_proj", "model.layers.82.mlp.experts.99.up_proj", "model.layers.82.mlp.experts.100.up_proj", "model.layers.82.mlp.experts.101.up_proj", "model.layers.82.mlp.experts.102.up_proj", "model.layers.82.mlp.experts.103.up_proj", "model.layers.82.mlp.experts.104.up_proj", "model.layers.82.mlp.experts.105.up_proj", "model.layers.82.mlp.experts.106.up_proj", "model.layers.82.mlp.experts.107.up_proj", "model.layers.82.mlp.experts.108.up_proj", "model.layers.82.mlp.experts.109.up_proj", "model.layers.82.mlp.experts.110.up_proj", "model.layers.82.mlp.experts.111.up_proj", "model.layers.82.mlp.experts.112.up_proj", "model.layers.82.mlp.experts.113.up_proj", "model.layers.82.mlp.experts.114.up_proj", "model.layers.82.mlp.experts.115.up_proj", "model.layers.82.mlp.experts.116.up_proj", "model.layers.82.mlp.experts.117.up_proj", "model.layers.82.mlp.experts.118.up_proj", "model.layers.82.mlp.experts.119.up_proj", "model.layers.82.mlp.experts.120.up_proj", "model.layers.82.mlp.experts.121.up_proj", "model.layers.82.mlp.experts.122.up_proj", "model.layers.82.mlp.experts.123.up_proj", "model.layers.82.mlp.experts.124.up_proj", "model.layers.82.mlp.experts.125.up_proj", "model.layers.82.mlp.experts.126.up_proj", "model.layers.82.mlp.experts.127.up_proj", "model.layers.82.mlp.experts.128.up_proj", "model.layers.82.mlp.experts.129.up_proj", "model.layers.82.mlp.experts.130.up_proj", "model.layers.82.mlp.experts.131.up_proj", "model.layers.82.mlp.experts.132.up_proj", "model.layers.82.mlp.experts.133.up_proj", "model.layers.82.mlp.experts.134.up_proj", "model.layers.82.mlp.experts.135.up_proj", "model.layers.82.mlp.experts.136.up_proj", "model.layers.82.mlp.experts.137.up_proj", "model.layers.82.mlp.experts.138.up_proj", "model.layers.82.mlp.experts.139.up_proj", "model.layers.82.mlp.experts.140.up_proj", "model.layers.82.mlp.experts.141.up_proj", "model.layers.82.mlp.experts.142.up_proj", "model.layers.82.mlp.experts.143.up_proj", "model.layers.82.mlp.experts.144.up_proj", "model.layers.82.mlp.experts.145.up_proj", "model.layers.82.mlp.experts.146.up_proj", "model.layers.82.mlp.experts.147.up_proj", "model.layers.82.mlp.experts.148.up_proj", "model.layers.82.mlp.experts.149.up_proj", "model.layers.82.mlp.experts.150.up_proj", "model.layers.82.mlp.experts.151.up_proj", "model.layers.82.mlp.experts.152.up_proj", "model.layers.82.mlp.experts.153.up_proj", "model.layers.82.mlp.experts.154.up_proj", "model.layers.82.mlp.experts.155.up_proj", "model.layers.82.mlp.experts.156.up_proj", "model.layers.82.mlp.experts.157.up_proj", "model.layers.82.mlp.experts.158.up_proj", "model.layers.82.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.871494067833044e-07, "dbits": 5033164800 } ] }, { "idx": 494, "layers": [ "model.layers.82.mlp.experts.0.down_proj", "model.layers.82.mlp.experts.1.down_proj", "model.layers.82.mlp.experts.2.down_proj", "model.layers.82.mlp.experts.3.down_proj", "model.layers.82.mlp.experts.4.down_proj", "model.layers.82.mlp.experts.5.down_proj", "model.layers.82.mlp.experts.6.down_proj", "model.layers.82.mlp.experts.7.down_proj", "model.layers.82.mlp.experts.8.down_proj", "model.layers.82.mlp.experts.9.down_proj", "model.layers.82.mlp.experts.10.down_proj", "model.layers.82.mlp.experts.11.down_proj", "model.layers.82.mlp.experts.12.down_proj", "model.layers.82.mlp.experts.13.down_proj", "model.layers.82.mlp.experts.14.down_proj", "model.layers.82.mlp.experts.15.down_proj", "model.layers.82.mlp.experts.16.down_proj", "model.layers.82.mlp.experts.17.down_proj", "model.layers.82.mlp.experts.18.down_proj", "model.layers.82.mlp.experts.19.down_proj", "model.layers.82.mlp.experts.20.down_proj", "model.layers.82.mlp.experts.21.down_proj", "model.layers.82.mlp.experts.22.down_proj", "model.layers.82.mlp.experts.23.down_proj", "model.layers.82.mlp.experts.24.down_proj", "model.layers.82.mlp.experts.25.down_proj", "model.layers.82.mlp.experts.26.down_proj", "model.layers.82.mlp.experts.27.down_proj", "model.layers.82.mlp.experts.28.down_proj", "model.layers.82.mlp.experts.29.down_proj", "model.layers.82.mlp.experts.30.down_proj", "model.layers.82.mlp.experts.31.down_proj", "model.layers.82.mlp.experts.32.down_proj", "model.layers.82.mlp.experts.33.down_proj", "model.layers.82.mlp.experts.34.down_proj", "model.layers.82.mlp.experts.35.down_proj", "model.layers.82.mlp.experts.36.down_proj", "model.layers.82.mlp.experts.37.down_proj", "model.layers.82.mlp.experts.38.down_proj", "model.layers.82.mlp.experts.39.down_proj", "model.layers.82.mlp.experts.40.down_proj", "model.layers.82.mlp.experts.41.down_proj", "model.layers.82.mlp.experts.42.down_proj", "model.layers.82.mlp.experts.43.down_proj", "model.layers.82.mlp.experts.44.down_proj", "model.layers.82.mlp.experts.45.down_proj", "model.layers.82.mlp.experts.46.down_proj", "model.layers.82.mlp.experts.47.down_proj", "model.layers.82.mlp.experts.48.down_proj", "model.layers.82.mlp.experts.49.down_proj", "model.layers.82.mlp.experts.50.down_proj", "model.layers.82.mlp.experts.51.down_proj", "model.layers.82.mlp.experts.52.down_proj", "model.layers.82.mlp.experts.53.down_proj", "model.layers.82.mlp.experts.54.down_proj", "model.layers.82.mlp.experts.55.down_proj", "model.layers.82.mlp.experts.56.down_proj", "model.layers.82.mlp.experts.57.down_proj", "model.layers.82.mlp.experts.58.down_proj", "model.layers.82.mlp.experts.59.down_proj", "model.layers.82.mlp.experts.60.down_proj", "model.layers.82.mlp.experts.61.down_proj", "model.layers.82.mlp.experts.62.down_proj", "model.layers.82.mlp.experts.63.down_proj", "model.layers.82.mlp.experts.64.down_proj", "model.layers.82.mlp.experts.65.down_proj", "model.layers.82.mlp.experts.66.down_proj", "model.layers.82.mlp.experts.67.down_proj", "model.layers.82.mlp.experts.68.down_proj", "model.layers.82.mlp.experts.69.down_proj", "model.layers.82.mlp.experts.70.down_proj", "model.layers.82.mlp.experts.71.down_proj", "model.layers.82.mlp.experts.72.down_proj", "model.layers.82.mlp.experts.73.down_proj", "model.layers.82.mlp.experts.74.down_proj", "model.layers.82.mlp.experts.75.down_proj", "model.layers.82.mlp.experts.76.down_proj", "model.layers.82.mlp.experts.77.down_proj", "model.layers.82.mlp.experts.78.down_proj", "model.layers.82.mlp.experts.79.down_proj", "model.layers.82.mlp.experts.80.down_proj", "model.layers.82.mlp.experts.81.down_proj", "model.layers.82.mlp.experts.82.down_proj", "model.layers.82.mlp.experts.83.down_proj", "model.layers.82.mlp.experts.84.down_proj", "model.layers.82.mlp.experts.85.down_proj", "model.layers.82.mlp.experts.86.down_proj", "model.layers.82.mlp.experts.87.down_proj", "model.layers.82.mlp.experts.88.down_proj", "model.layers.82.mlp.experts.89.down_proj", "model.layers.82.mlp.experts.90.down_proj", "model.layers.82.mlp.experts.91.down_proj", "model.layers.82.mlp.experts.92.down_proj", "model.layers.82.mlp.experts.93.down_proj", "model.layers.82.mlp.experts.94.down_proj", "model.layers.82.mlp.experts.95.down_proj", "model.layers.82.mlp.experts.96.down_proj", "model.layers.82.mlp.experts.97.down_proj", "model.layers.82.mlp.experts.98.down_proj", "model.layers.82.mlp.experts.99.down_proj", "model.layers.82.mlp.experts.100.down_proj", "model.layers.82.mlp.experts.101.down_proj", "model.layers.82.mlp.experts.102.down_proj", "model.layers.82.mlp.experts.103.down_proj", "model.layers.82.mlp.experts.104.down_proj", "model.layers.82.mlp.experts.105.down_proj", "model.layers.82.mlp.experts.106.down_proj", "model.layers.82.mlp.experts.107.down_proj", "model.layers.82.mlp.experts.108.down_proj", "model.layers.82.mlp.experts.109.down_proj", "model.layers.82.mlp.experts.110.down_proj", "model.layers.82.mlp.experts.111.down_proj", "model.layers.82.mlp.experts.112.down_proj", "model.layers.82.mlp.experts.113.down_proj", "model.layers.82.mlp.experts.114.down_proj", "model.layers.82.mlp.experts.115.down_proj", "model.layers.82.mlp.experts.116.down_proj", "model.layers.82.mlp.experts.117.down_proj", "model.layers.82.mlp.experts.118.down_proj", "model.layers.82.mlp.experts.119.down_proj", "model.layers.82.mlp.experts.120.down_proj", "model.layers.82.mlp.experts.121.down_proj", "model.layers.82.mlp.experts.122.down_proj", "model.layers.82.mlp.experts.123.down_proj", "model.layers.82.mlp.experts.124.down_proj", "model.layers.82.mlp.experts.125.down_proj", "model.layers.82.mlp.experts.126.down_proj", "model.layers.82.mlp.experts.127.down_proj", "model.layers.82.mlp.experts.128.down_proj", "model.layers.82.mlp.experts.129.down_proj", "model.layers.82.mlp.experts.130.down_proj", "model.layers.82.mlp.experts.131.down_proj", "model.layers.82.mlp.experts.132.down_proj", "model.layers.82.mlp.experts.133.down_proj", "model.layers.82.mlp.experts.134.down_proj", "model.layers.82.mlp.experts.135.down_proj", "model.layers.82.mlp.experts.136.down_proj", "model.layers.82.mlp.experts.137.down_proj", "model.layers.82.mlp.experts.138.down_proj", "model.layers.82.mlp.experts.139.down_proj", "model.layers.82.mlp.experts.140.down_proj", "model.layers.82.mlp.experts.141.down_proj", "model.layers.82.mlp.experts.142.down_proj", "model.layers.82.mlp.experts.143.down_proj", "model.layers.82.mlp.experts.144.down_proj", "model.layers.82.mlp.experts.145.down_proj", "model.layers.82.mlp.experts.146.down_proj", "model.layers.82.mlp.experts.147.down_proj", "model.layers.82.mlp.experts.148.down_proj", "model.layers.82.mlp.experts.149.down_proj", "model.layers.82.mlp.experts.150.down_proj", "model.layers.82.mlp.experts.151.down_proj", "model.layers.82.mlp.experts.152.down_proj", "model.layers.82.mlp.experts.153.down_proj", "model.layers.82.mlp.experts.154.down_proj", "model.layers.82.mlp.experts.155.down_proj", "model.layers.82.mlp.experts.156.down_proj", "model.layers.82.mlp.experts.157.down_proj", "model.layers.82.mlp.experts.158.down_proj", "model.layers.82.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 3.2363808713853142e-06, "dbits": 2516582400 } ] }, { "idx": 495, "layers": [ "model.layers.83.self_attn.q_proj" ], "candidates": [ { "dkld": -6.284244591370225e-07, "dbits": 125829120 } ] }, { "idx": 496, "layers": [ "model.layers.83.self_attn.k_proj", "model.layers.83.self_attn.v_proj" ], "candidates": [ { "dkld": 8.973409421741529e-07, "dbits": 20971520 } ] }, { "idx": 497, "layers": [ "model.layers.83.self_attn.o_proj" ], "candidates": [ { "dkld": -2.6866851840168455e-06, "dbits": 125829120 } ] }, { "idx": 498, "layers": [ "model.layers.83.mlp.shared_experts.gate_proj", "model.layers.83.mlp.shared_experts.up_proj", "model.layers.83.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.0633639805019336e-06, "dbits": 47185920 } ] }, { "idx": 499, "layers": [ "model.layers.83.mlp.experts.0.gate_proj", "model.layers.83.mlp.experts.1.gate_proj", "model.layers.83.mlp.experts.2.gate_proj", "model.layers.83.mlp.experts.3.gate_proj", "model.layers.83.mlp.experts.4.gate_proj", "model.layers.83.mlp.experts.5.gate_proj", "model.layers.83.mlp.experts.6.gate_proj", "model.layers.83.mlp.experts.7.gate_proj", "model.layers.83.mlp.experts.8.gate_proj", "model.layers.83.mlp.experts.9.gate_proj", "model.layers.83.mlp.experts.10.gate_proj", "model.layers.83.mlp.experts.11.gate_proj", "model.layers.83.mlp.experts.12.gate_proj", "model.layers.83.mlp.experts.13.gate_proj", "model.layers.83.mlp.experts.14.gate_proj", "model.layers.83.mlp.experts.15.gate_proj", "model.layers.83.mlp.experts.16.gate_proj", "model.layers.83.mlp.experts.17.gate_proj", "model.layers.83.mlp.experts.18.gate_proj", "model.layers.83.mlp.experts.19.gate_proj", "model.layers.83.mlp.experts.20.gate_proj", "model.layers.83.mlp.experts.21.gate_proj", "model.layers.83.mlp.experts.22.gate_proj", "model.layers.83.mlp.experts.23.gate_proj", "model.layers.83.mlp.experts.24.gate_proj", "model.layers.83.mlp.experts.25.gate_proj", "model.layers.83.mlp.experts.26.gate_proj", "model.layers.83.mlp.experts.27.gate_proj", "model.layers.83.mlp.experts.28.gate_proj", "model.layers.83.mlp.experts.29.gate_proj", "model.layers.83.mlp.experts.30.gate_proj", "model.layers.83.mlp.experts.31.gate_proj", "model.layers.83.mlp.experts.32.gate_proj", "model.layers.83.mlp.experts.33.gate_proj", "model.layers.83.mlp.experts.34.gate_proj", "model.layers.83.mlp.experts.35.gate_proj", "model.layers.83.mlp.experts.36.gate_proj", "model.layers.83.mlp.experts.37.gate_proj", "model.layers.83.mlp.experts.38.gate_proj", "model.layers.83.mlp.experts.39.gate_proj", "model.layers.83.mlp.experts.40.gate_proj", "model.layers.83.mlp.experts.41.gate_proj", "model.layers.83.mlp.experts.42.gate_proj", "model.layers.83.mlp.experts.43.gate_proj", "model.layers.83.mlp.experts.44.gate_proj", "model.layers.83.mlp.experts.45.gate_proj", "model.layers.83.mlp.experts.46.gate_proj", "model.layers.83.mlp.experts.47.gate_proj", "model.layers.83.mlp.experts.48.gate_proj", "model.layers.83.mlp.experts.49.gate_proj", "model.layers.83.mlp.experts.50.gate_proj", "model.layers.83.mlp.experts.51.gate_proj", "model.layers.83.mlp.experts.52.gate_proj", "model.layers.83.mlp.experts.53.gate_proj", "model.layers.83.mlp.experts.54.gate_proj", "model.layers.83.mlp.experts.55.gate_proj", "model.layers.83.mlp.experts.56.gate_proj", "model.layers.83.mlp.experts.57.gate_proj", "model.layers.83.mlp.experts.58.gate_proj", "model.layers.83.mlp.experts.59.gate_proj", "model.layers.83.mlp.experts.60.gate_proj", "model.layers.83.mlp.experts.61.gate_proj", "model.layers.83.mlp.experts.62.gate_proj", "model.layers.83.mlp.experts.63.gate_proj", "model.layers.83.mlp.experts.64.gate_proj", "model.layers.83.mlp.experts.65.gate_proj", "model.layers.83.mlp.experts.66.gate_proj", "model.layers.83.mlp.experts.67.gate_proj", "model.layers.83.mlp.experts.68.gate_proj", "model.layers.83.mlp.experts.69.gate_proj", "model.layers.83.mlp.experts.70.gate_proj", "model.layers.83.mlp.experts.71.gate_proj", "model.layers.83.mlp.experts.72.gate_proj", "model.layers.83.mlp.experts.73.gate_proj", "model.layers.83.mlp.experts.74.gate_proj", "model.layers.83.mlp.experts.75.gate_proj", "model.layers.83.mlp.experts.76.gate_proj", "model.layers.83.mlp.experts.77.gate_proj", "model.layers.83.mlp.experts.78.gate_proj", "model.layers.83.mlp.experts.79.gate_proj", "model.layers.83.mlp.experts.80.gate_proj", "model.layers.83.mlp.experts.81.gate_proj", "model.layers.83.mlp.experts.82.gate_proj", "model.layers.83.mlp.experts.83.gate_proj", "model.layers.83.mlp.experts.84.gate_proj", "model.layers.83.mlp.experts.85.gate_proj", "model.layers.83.mlp.experts.86.gate_proj", "model.layers.83.mlp.experts.87.gate_proj", "model.layers.83.mlp.experts.88.gate_proj", "model.layers.83.mlp.experts.89.gate_proj", "model.layers.83.mlp.experts.90.gate_proj", "model.layers.83.mlp.experts.91.gate_proj", "model.layers.83.mlp.experts.92.gate_proj", "model.layers.83.mlp.experts.93.gate_proj", "model.layers.83.mlp.experts.94.gate_proj", "model.layers.83.mlp.experts.95.gate_proj", "model.layers.83.mlp.experts.96.gate_proj", "model.layers.83.mlp.experts.97.gate_proj", "model.layers.83.mlp.experts.98.gate_proj", "model.layers.83.mlp.experts.99.gate_proj", "model.layers.83.mlp.experts.100.gate_proj", "model.layers.83.mlp.experts.101.gate_proj", "model.layers.83.mlp.experts.102.gate_proj", "model.layers.83.mlp.experts.103.gate_proj", "model.layers.83.mlp.experts.104.gate_proj", "model.layers.83.mlp.experts.105.gate_proj", "model.layers.83.mlp.experts.106.gate_proj", "model.layers.83.mlp.experts.107.gate_proj", "model.layers.83.mlp.experts.108.gate_proj", "model.layers.83.mlp.experts.109.gate_proj", "model.layers.83.mlp.experts.110.gate_proj", "model.layers.83.mlp.experts.111.gate_proj", "model.layers.83.mlp.experts.112.gate_proj", "model.layers.83.mlp.experts.113.gate_proj", "model.layers.83.mlp.experts.114.gate_proj", "model.layers.83.mlp.experts.115.gate_proj", "model.layers.83.mlp.experts.116.gate_proj", "model.layers.83.mlp.experts.117.gate_proj", "model.layers.83.mlp.experts.118.gate_proj", "model.layers.83.mlp.experts.119.gate_proj", "model.layers.83.mlp.experts.120.gate_proj", "model.layers.83.mlp.experts.121.gate_proj", "model.layers.83.mlp.experts.122.gate_proj", "model.layers.83.mlp.experts.123.gate_proj", "model.layers.83.mlp.experts.124.gate_proj", "model.layers.83.mlp.experts.125.gate_proj", "model.layers.83.mlp.experts.126.gate_proj", "model.layers.83.mlp.experts.127.gate_proj", "model.layers.83.mlp.experts.128.gate_proj", "model.layers.83.mlp.experts.129.gate_proj", "model.layers.83.mlp.experts.130.gate_proj", "model.layers.83.mlp.experts.131.gate_proj", "model.layers.83.mlp.experts.132.gate_proj", "model.layers.83.mlp.experts.133.gate_proj", "model.layers.83.mlp.experts.134.gate_proj", "model.layers.83.mlp.experts.135.gate_proj", "model.layers.83.mlp.experts.136.gate_proj", "model.layers.83.mlp.experts.137.gate_proj", "model.layers.83.mlp.experts.138.gate_proj", "model.layers.83.mlp.experts.139.gate_proj", "model.layers.83.mlp.experts.140.gate_proj", "model.layers.83.mlp.experts.141.gate_proj", "model.layers.83.mlp.experts.142.gate_proj", "model.layers.83.mlp.experts.143.gate_proj", "model.layers.83.mlp.experts.144.gate_proj", "model.layers.83.mlp.experts.145.gate_proj", "model.layers.83.mlp.experts.146.gate_proj", "model.layers.83.mlp.experts.147.gate_proj", "model.layers.83.mlp.experts.148.gate_proj", "model.layers.83.mlp.experts.149.gate_proj", "model.layers.83.mlp.experts.150.gate_proj", "model.layers.83.mlp.experts.151.gate_proj", "model.layers.83.mlp.experts.152.gate_proj", "model.layers.83.mlp.experts.153.gate_proj", "model.layers.83.mlp.experts.154.gate_proj", "model.layers.83.mlp.experts.155.gate_proj", "model.layers.83.mlp.experts.156.gate_proj", "model.layers.83.mlp.experts.157.gate_proj", "model.layers.83.mlp.experts.158.gate_proj", "model.layers.83.mlp.experts.159.gate_proj", "model.layers.83.mlp.experts.0.up_proj", "model.layers.83.mlp.experts.1.up_proj", "model.layers.83.mlp.experts.2.up_proj", "model.layers.83.mlp.experts.3.up_proj", "model.layers.83.mlp.experts.4.up_proj", "model.layers.83.mlp.experts.5.up_proj", "model.layers.83.mlp.experts.6.up_proj", "model.layers.83.mlp.experts.7.up_proj", "model.layers.83.mlp.experts.8.up_proj", "model.layers.83.mlp.experts.9.up_proj", "model.layers.83.mlp.experts.10.up_proj", "model.layers.83.mlp.experts.11.up_proj", "model.layers.83.mlp.experts.12.up_proj", "model.layers.83.mlp.experts.13.up_proj", "model.layers.83.mlp.experts.14.up_proj", "model.layers.83.mlp.experts.15.up_proj", "model.layers.83.mlp.experts.16.up_proj", "model.layers.83.mlp.experts.17.up_proj", "model.layers.83.mlp.experts.18.up_proj", "model.layers.83.mlp.experts.19.up_proj", "model.layers.83.mlp.experts.20.up_proj", "model.layers.83.mlp.experts.21.up_proj", "model.layers.83.mlp.experts.22.up_proj", "model.layers.83.mlp.experts.23.up_proj", "model.layers.83.mlp.experts.24.up_proj", "model.layers.83.mlp.experts.25.up_proj", "model.layers.83.mlp.experts.26.up_proj", "model.layers.83.mlp.experts.27.up_proj", "model.layers.83.mlp.experts.28.up_proj", "model.layers.83.mlp.experts.29.up_proj", "model.layers.83.mlp.experts.30.up_proj", "model.layers.83.mlp.experts.31.up_proj", "model.layers.83.mlp.experts.32.up_proj", "model.layers.83.mlp.experts.33.up_proj", "model.layers.83.mlp.experts.34.up_proj", "model.layers.83.mlp.experts.35.up_proj", "model.layers.83.mlp.experts.36.up_proj", "model.layers.83.mlp.experts.37.up_proj", "model.layers.83.mlp.experts.38.up_proj", "model.layers.83.mlp.experts.39.up_proj", "model.layers.83.mlp.experts.40.up_proj", "model.layers.83.mlp.experts.41.up_proj", "model.layers.83.mlp.experts.42.up_proj", "model.layers.83.mlp.experts.43.up_proj", "model.layers.83.mlp.experts.44.up_proj", "model.layers.83.mlp.experts.45.up_proj", "model.layers.83.mlp.experts.46.up_proj", "model.layers.83.mlp.experts.47.up_proj", "model.layers.83.mlp.experts.48.up_proj", "model.layers.83.mlp.experts.49.up_proj", "model.layers.83.mlp.experts.50.up_proj", "model.layers.83.mlp.experts.51.up_proj", "model.layers.83.mlp.experts.52.up_proj", "model.layers.83.mlp.experts.53.up_proj", "model.layers.83.mlp.experts.54.up_proj", "model.layers.83.mlp.experts.55.up_proj", "model.layers.83.mlp.experts.56.up_proj", "model.layers.83.mlp.experts.57.up_proj", "model.layers.83.mlp.experts.58.up_proj", "model.layers.83.mlp.experts.59.up_proj", "model.layers.83.mlp.experts.60.up_proj", "model.layers.83.mlp.experts.61.up_proj", "model.layers.83.mlp.experts.62.up_proj", "model.layers.83.mlp.experts.63.up_proj", "model.layers.83.mlp.experts.64.up_proj", "model.layers.83.mlp.experts.65.up_proj", "model.layers.83.mlp.experts.66.up_proj", "model.layers.83.mlp.experts.67.up_proj", "model.layers.83.mlp.experts.68.up_proj", "model.layers.83.mlp.experts.69.up_proj", "model.layers.83.mlp.experts.70.up_proj", "model.layers.83.mlp.experts.71.up_proj", "model.layers.83.mlp.experts.72.up_proj", "model.layers.83.mlp.experts.73.up_proj", "model.layers.83.mlp.experts.74.up_proj", "model.layers.83.mlp.experts.75.up_proj", "model.layers.83.mlp.experts.76.up_proj", "model.layers.83.mlp.experts.77.up_proj", "model.layers.83.mlp.experts.78.up_proj", "model.layers.83.mlp.experts.79.up_proj", "model.layers.83.mlp.experts.80.up_proj", "model.layers.83.mlp.experts.81.up_proj", "model.layers.83.mlp.experts.82.up_proj", "model.layers.83.mlp.experts.83.up_proj", "model.layers.83.mlp.experts.84.up_proj", "model.layers.83.mlp.experts.85.up_proj", "model.layers.83.mlp.experts.86.up_proj", "model.layers.83.mlp.experts.87.up_proj", "model.layers.83.mlp.experts.88.up_proj", "model.layers.83.mlp.experts.89.up_proj", "model.layers.83.mlp.experts.90.up_proj", "model.layers.83.mlp.experts.91.up_proj", "model.layers.83.mlp.experts.92.up_proj", "model.layers.83.mlp.experts.93.up_proj", "model.layers.83.mlp.experts.94.up_proj", "model.layers.83.mlp.experts.95.up_proj", "model.layers.83.mlp.experts.96.up_proj", "model.layers.83.mlp.experts.97.up_proj", "model.layers.83.mlp.experts.98.up_proj", "model.layers.83.mlp.experts.99.up_proj", "model.layers.83.mlp.experts.100.up_proj", "model.layers.83.mlp.experts.101.up_proj", "model.layers.83.mlp.experts.102.up_proj", "model.layers.83.mlp.experts.103.up_proj", "model.layers.83.mlp.experts.104.up_proj", "model.layers.83.mlp.experts.105.up_proj", "model.layers.83.mlp.experts.106.up_proj", "model.layers.83.mlp.experts.107.up_proj", "model.layers.83.mlp.experts.108.up_proj", "model.layers.83.mlp.experts.109.up_proj", "model.layers.83.mlp.experts.110.up_proj", "model.layers.83.mlp.experts.111.up_proj", "model.layers.83.mlp.experts.112.up_proj", "model.layers.83.mlp.experts.113.up_proj", "model.layers.83.mlp.experts.114.up_proj", "model.layers.83.mlp.experts.115.up_proj", "model.layers.83.mlp.experts.116.up_proj", "model.layers.83.mlp.experts.117.up_proj", "model.layers.83.mlp.experts.118.up_proj", "model.layers.83.mlp.experts.119.up_proj", "model.layers.83.mlp.experts.120.up_proj", "model.layers.83.mlp.experts.121.up_proj", "model.layers.83.mlp.experts.122.up_proj", "model.layers.83.mlp.experts.123.up_proj", "model.layers.83.mlp.experts.124.up_proj", "model.layers.83.mlp.experts.125.up_proj", "model.layers.83.mlp.experts.126.up_proj", "model.layers.83.mlp.experts.127.up_proj", "model.layers.83.mlp.experts.128.up_proj", "model.layers.83.mlp.experts.129.up_proj", "model.layers.83.mlp.experts.130.up_proj", "model.layers.83.mlp.experts.131.up_proj", "model.layers.83.mlp.experts.132.up_proj", "model.layers.83.mlp.experts.133.up_proj", "model.layers.83.mlp.experts.134.up_proj", "model.layers.83.mlp.experts.135.up_proj", "model.layers.83.mlp.experts.136.up_proj", "model.layers.83.mlp.experts.137.up_proj", "model.layers.83.mlp.experts.138.up_proj", "model.layers.83.mlp.experts.139.up_proj", "model.layers.83.mlp.experts.140.up_proj", "model.layers.83.mlp.experts.141.up_proj", "model.layers.83.mlp.experts.142.up_proj", "model.layers.83.mlp.experts.143.up_proj", "model.layers.83.mlp.experts.144.up_proj", "model.layers.83.mlp.experts.145.up_proj", "model.layers.83.mlp.experts.146.up_proj", "model.layers.83.mlp.experts.147.up_proj", "model.layers.83.mlp.experts.148.up_proj", "model.layers.83.mlp.experts.149.up_proj", "model.layers.83.mlp.experts.150.up_proj", "model.layers.83.mlp.experts.151.up_proj", "model.layers.83.mlp.experts.152.up_proj", "model.layers.83.mlp.experts.153.up_proj", "model.layers.83.mlp.experts.154.up_proj", "model.layers.83.mlp.experts.155.up_proj", "model.layers.83.mlp.experts.156.up_proj", "model.layers.83.mlp.experts.157.up_proj", "model.layers.83.mlp.experts.158.up_proj", "model.layers.83.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.3871758710595154e-07, "dbits": 5033164800 } ] }, { "idx": 500, "layers": [ "model.layers.83.mlp.experts.0.down_proj", "model.layers.83.mlp.experts.1.down_proj", "model.layers.83.mlp.experts.2.down_proj", "model.layers.83.mlp.experts.3.down_proj", "model.layers.83.mlp.experts.4.down_proj", "model.layers.83.mlp.experts.5.down_proj", "model.layers.83.mlp.experts.6.down_proj", "model.layers.83.mlp.experts.7.down_proj", "model.layers.83.mlp.experts.8.down_proj", "model.layers.83.mlp.experts.9.down_proj", "model.layers.83.mlp.experts.10.down_proj", "model.layers.83.mlp.experts.11.down_proj", "model.layers.83.mlp.experts.12.down_proj", "model.layers.83.mlp.experts.13.down_proj", "model.layers.83.mlp.experts.14.down_proj", "model.layers.83.mlp.experts.15.down_proj", "model.layers.83.mlp.experts.16.down_proj", "model.layers.83.mlp.experts.17.down_proj", "model.layers.83.mlp.experts.18.down_proj", "model.layers.83.mlp.experts.19.down_proj", "model.layers.83.mlp.experts.20.down_proj", "model.layers.83.mlp.experts.21.down_proj", "model.layers.83.mlp.experts.22.down_proj", "model.layers.83.mlp.experts.23.down_proj", "model.layers.83.mlp.experts.24.down_proj", "model.layers.83.mlp.experts.25.down_proj", "model.layers.83.mlp.experts.26.down_proj", "model.layers.83.mlp.experts.27.down_proj", "model.layers.83.mlp.experts.28.down_proj", "model.layers.83.mlp.experts.29.down_proj", "model.layers.83.mlp.experts.30.down_proj", "model.layers.83.mlp.experts.31.down_proj", "model.layers.83.mlp.experts.32.down_proj", "model.layers.83.mlp.experts.33.down_proj", "model.layers.83.mlp.experts.34.down_proj", "model.layers.83.mlp.experts.35.down_proj", "model.layers.83.mlp.experts.36.down_proj", "model.layers.83.mlp.experts.37.down_proj", "model.layers.83.mlp.experts.38.down_proj", "model.layers.83.mlp.experts.39.down_proj", "model.layers.83.mlp.experts.40.down_proj", "model.layers.83.mlp.experts.41.down_proj", "model.layers.83.mlp.experts.42.down_proj", "model.layers.83.mlp.experts.43.down_proj", "model.layers.83.mlp.experts.44.down_proj", "model.layers.83.mlp.experts.45.down_proj", "model.layers.83.mlp.experts.46.down_proj", "model.layers.83.mlp.experts.47.down_proj", "model.layers.83.mlp.experts.48.down_proj", "model.layers.83.mlp.experts.49.down_proj", "model.layers.83.mlp.experts.50.down_proj", "model.layers.83.mlp.experts.51.down_proj", "model.layers.83.mlp.experts.52.down_proj", "model.layers.83.mlp.experts.53.down_proj", "model.layers.83.mlp.experts.54.down_proj", "model.layers.83.mlp.experts.55.down_proj", "model.layers.83.mlp.experts.56.down_proj", "model.layers.83.mlp.experts.57.down_proj", "model.layers.83.mlp.experts.58.down_proj", "model.layers.83.mlp.experts.59.down_proj", "model.layers.83.mlp.experts.60.down_proj", "model.layers.83.mlp.experts.61.down_proj", "model.layers.83.mlp.experts.62.down_proj", "model.layers.83.mlp.experts.63.down_proj", "model.layers.83.mlp.experts.64.down_proj", "model.layers.83.mlp.experts.65.down_proj", "model.layers.83.mlp.experts.66.down_proj", "model.layers.83.mlp.experts.67.down_proj", "model.layers.83.mlp.experts.68.down_proj", "model.layers.83.mlp.experts.69.down_proj", "model.layers.83.mlp.experts.70.down_proj", "model.layers.83.mlp.experts.71.down_proj", "model.layers.83.mlp.experts.72.down_proj", "model.layers.83.mlp.experts.73.down_proj", "model.layers.83.mlp.experts.74.down_proj", "model.layers.83.mlp.experts.75.down_proj", "model.layers.83.mlp.experts.76.down_proj", "model.layers.83.mlp.experts.77.down_proj", "model.layers.83.mlp.experts.78.down_proj", "model.layers.83.mlp.experts.79.down_proj", "model.layers.83.mlp.experts.80.down_proj", "model.layers.83.mlp.experts.81.down_proj", "model.layers.83.mlp.experts.82.down_proj", "model.layers.83.mlp.experts.83.down_proj", "model.layers.83.mlp.experts.84.down_proj", "model.layers.83.mlp.experts.85.down_proj", "model.layers.83.mlp.experts.86.down_proj", "model.layers.83.mlp.experts.87.down_proj", "model.layers.83.mlp.experts.88.down_proj", "model.layers.83.mlp.experts.89.down_proj", "model.layers.83.mlp.experts.90.down_proj", "model.layers.83.mlp.experts.91.down_proj", "model.layers.83.mlp.experts.92.down_proj", "model.layers.83.mlp.experts.93.down_proj", "model.layers.83.mlp.experts.94.down_proj", "model.layers.83.mlp.experts.95.down_proj", "model.layers.83.mlp.experts.96.down_proj", "model.layers.83.mlp.experts.97.down_proj", "model.layers.83.mlp.experts.98.down_proj", "model.layers.83.mlp.experts.99.down_proj", "model.layers.83.mlp.experts.100.down_proj", "model.layers.83.mlp.experts.101.down_proj", "model.layers.83.mlp.experts.102.down_proj", "model.layers.83.mlp.experts.103.down_proj", "model.layers.83.mlp.experts.104.down_proj", "model.layers.83.mlp.experts.105.down_proj", "model.layers.83.mlp.experts.106.down_proj", "model.layers.83.mlp.experts.107.down_proj", "model.layers.83.mlp.experts.108.down_proj", "model.layers.83.mlp.experts.109.down_proj", "model.layers.83.mlp.experts.110.down_proj", "model.layers.83.mlp.experts.111.down_proj", "model.layers.83.mlp.experts.112.down_proj", "model.layers.83.mlp.experts.113.down_proj", "model.layers.83.mlp.experts.114.down_proj", "model.layers.83.mlp.experts.115.down_proj", "model.layers.83.mlp.experts.116.down_proj", "model.layers.83.mlp.experts.117.down_proj", "model.layers.83.mlp.experts.118.down_proj", "model.layers.83.mlp.experts.119.down_proj", "model.layers.83.mlp.experts.120.down_proj", "model.layers.83.mlp.experts.121.down_proj", "model.layers.83.mlp.experts.122.down_proj", "model.layers.83.mlp.experts.123.down_proj", "model.layers.83.mlp.experts.124.down_proj", "model.layers.83.mlp.experts.125.down_proj", "model.layers.83.mlp.experts.126.down_proj", "model.layers.83.mlp.experts.127.down_proj", "model.layers.83.mlp.experts.128.down_proj", "model.layers.83.mlp.experts.129.down_proj", "model.layers.83.mlp.experts.130.down_proj", "model.layers.83.mlp.experts.131.down_proj", "model.layers.83.mlp.experts.132.down_proj", "model.layers.83.mlp.experts.133.down_proj", "model.layers.83.mlp.experts.134.down_proj", "model.layers.83.mlp.experts.135.down_proj", "model.layers.83.mlp.experts.136.down_proj", "model.layers.83.mlp.experts.137.down_proj", "model.layers.83.mlp.experts.138.down_proj", "model.layers.83.mlp.experts.139.down_proj", "model.layers.83.mlp.experts.140.down_proj", "model.layers.83.mlp.experts.141.down_proj", "model.layers.83.mlp.experts.142.down_proj", "model.layers.83.mlp.experts.143.down_proj", "model.layers.83.mlp.experts.144.down_proj", "model.layers.83.mlp.experts.145.down_proj", "model.layers.83.mlp.experts.146.down_proj", "model.layers.83.mlp.experts.147.down_proj", "model.layers.83.mlp.experts.148.down_proj", "model.layers.83.mlp.experts.149.down_proj", "model.layers.83.mlp.experts.150.down_proj", "model.layers.83.mlp.experts.151.down_proj", "model.layers.83.mlp.experts.152.down_proj", "model.layers.83.mlp.experts.153.down_proj", "model.layers.83.mlp.experts.154.down_proj", "model.layers.83.mlp.experts.155.down_proj", "model.layers.83.mlp.experts.156.down_proj", "model.layers.83.mlp.experts.157.down_proj", "model.layers.83.mlp.experts.158.down_proj", "model.layers.83.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.5394383808597705e-06, "dbits": 2516582400 } ] }, { "idx": 501, "layers": [ "model.layers.84.self_attn.q_proj" ], "candidates": [ { "dkld": 3.172238939441762e-06, "dbits": 125829120 } ] }, { "idx": 502, "layers": [ "model.layers.84.self_attn.k_proj", "model.layers.84.self_attn.v_proj" ], "candidates": [ { "dkld": 3.4299679100513675e-06, "dbits": 20971520 } ] }, { "idx": 503, "layers": [ "model.layers.84.self_attn.o_proj" ], "candidates": [ { "dkld": -4.662867286242501e-06, "dbits": 125829120 } ] }, { "idx": 504, "layers": [ "model.layers.84.mlp.shared_experts.gate_proj", "model.layers.84.mlp.shared_experts.up_proj", "model.layers.84.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -3.814842784777284e-06, "dbits": 47185920 } ] }, { "idx": 505, "layers": [ "model.layers.84.mlp.experts.0.gate_proj", "model.layers.84.mlp.experts.1.gate_proj", "model.layers.84.mlp.experts.2.gate_proj", "model.layers.84.mlp.experts.3.gate_proj", "model.layers.84.mlp.experts.4.gate_proj", "model.layers.84.mlp.experts.5.gate_proj", "model.layers.84.mlp.experts.6.gate_proj", "model.layers.84.mlp.experts.7.gate_proj", "model.layers.84.mlp.experts.8.gate_proj", "model.layers.84.mlp.experts.9.gate_proj", "model.layers.84.mlp.experts.10.gate_proj", "model.layers.84.mlp.experts.11.gate_proj", "model.layers.84.mlp.experts.12.gate_proj", "model.layers.84.mlp.experts.13.gate_proj", "model.layers.84.mlp.experts.14.gate_proj", "model.layers.84.mlp.experts.15.gate_proj", "model.layers.84.mlp.experts.16.gate_proj", "model.layers.84.mlp.experts.17.gate_proj", "model.layers.84.mlp.experts.18.gate_proj", "model.layers.84.mlp.experts.19.gate_proj", "model.layers.84.mlp.experts.20.gate_proj", "model.layers.84.mlp.experts.21.gate_proj", "model.layers.84.mlp.experts.22.gate_proj", "model.layers.84.mlp.experts.23.gate_proj", "model.layers.84.mlp.experts.24.gate_proj", "model.layers.84.mlp.experts.25.gate_proj", "model.layers.84.mlp.experts.26.gate_proj", "model.layers.84.mlp.experts.27.gate_proj", "model.layers.84.mlp.experts.28.gate_proj", "model.layers.84.mlp.experts.29.gate_proj", "model.layers.84.mlp.experts.30.gate_proj", "model.layers.84.mlp.experts.31.gate_proj", "model.layers.84.mlp.experts.32.gate_proj", "model.layers.84.mlp.experts.33.gate_proj", "model.layers.84.mlp.experts.34.gate_proj", "model.layers.84.mlp.experts.35.gate_proj", "model.layers.84.mlp.experts.36.gate_proj", "model.layers.84.mlp.experts.37.gate_proj", "model.layers.84.mlp.experts.38.gate_proj", "model.layers.84.mlp.experts.39.gate_proj", "model.layers.84.mlp.experts.40.gate_proj", "model.layers.84.mlp.experts.41.gate_proj", "model.layers.84.mlp.experts.42.gate_proj", "model.layers.84.mlp.experts.43.gate_proj", "model.layers.84.mlp.experts.44.gate_proj", "model.layers.84.mlp.experts.45.gate_proj", "model.layers.84.mlp.experts.46.gate_proj", "model.layers.84.mlp.experts.47.gate_proj", "model.layers.84.mlp.experts.48.gate_proj", "model.layers.84.mlp.experts.49.gate_proj", "model.layers.84.mlp.experts.50.gate_proj", "model.layers.84.mlp.experts.51.gate_proj", "model.layers.84.mlp.experts.52.gate_proj", "model.layers.84.mlp.experts.53.gate_proj", "model.layers.84.mlp.experts.54.gate_proj", "model.layers.84.mlp.experts.55.gate_proj", "model.layers.84.mlp.experts.56.gate_proj", "model.layers.84.mlp.experts.57.gate_proj", "model.layers.84.mlp.experts.58.gate_proj", "model.layers.84.mlp.experts.59.gate_proj", "model.layers.84.mlp.experts.60.gate_proj", "model.layers.84.mlp.experts.61.gate_proj", "model.layers.84.mlp.experts.62.gate_proj", "model.layers.84.mlp.experts.63.gate_proj", "model.layers.84.mlp.experts.64.gate_proj", "model.layers.84.mlp.experts.65.gate_proj", "model.layers.84.mlp.experts.66.gate_proj", "model.layers.84.mlp.experts.67.gate_proj", "model.layers.84.mlp.experts.68.gate_proj", "model.layers.84.mlp.experts.69.gate_proj", "model.layers.84.mlp.experts.70.gate_proj", "model.layers.84.mlp.experts.71.gate_proj", "model.layers.84.mlp.experts.72.gate_proj", "model.layers.84.mlp.experts.73.gate_proj", "model.layers.84.mlp.experts.74.gate_proj", "model.layers.84.mlp.experts.75.gate_proj", "model.layers.84.mlp.experts.76.gate_proj", "model.layers.84.mlp.experts.77.gate_proj", "model.layers.84.mlp.experts.78.gate_proj", "model.layers.84.mlp.experts.79.gate_proj", "model.layers.84.mlp.experts.80.gate_proj", "model.layers.84.mlp.experts.81.gate_proj", "model.layers.84.mlp.experts.82.gate_proj", "model.layers.84.mlp.experts.83.gate_proj", "model.layers.84.mlp.experts.84.gate_proj", "model.layers.84.mlp.experts.85.gate_proj", "model.layers.84.mlp.experts.86.gate_proj", "model.layers.84.mlp.experts.87.gate_proj", "model.layers.84.mlp.experts.88.gate_proj", "model.layers.84.mlp.experts.89.gate_proj", "model.layers.84.mlp.experts.90.gate_proj", "model.layers.84.mlp.experts.91.gate_proj", "model.layers.84.mlp.experts.92.gate_proj", "model.layers.84.mlp.experts.93.gate_proj", "model.layers.84.mlp.experts.94.gate_proj", "model.layers.84.mlp.experts.95.gate_proj", "model.layers.84.mlp.experts.96.gate_proj", "model.layers.84.mlp.experts.97.gate_proj", "model.layers.84.mlp.experts.98.gate_proj", "model.layers.84.mlp.experts.99.gate_proj", "model.layers.84.mlp.experts.100.gate_proj", "model.layers.84.mlp.experts.101.gate_proj", "model.layers.84.mlp.experts.102.gate_proj", "model.layers.84.mlp.experts.103.gate_proj", "model.layers.84.mlp.experts.104.gate_proj", "model.layers.84.mlp.experts.105.gate_proj", "model.layers.84.mlp.experts.106.gate_proj", "model.layers.84.mlp.experts.107.gate_proj", "model.layers.84.mlp.experts.108.gate_proj", "model.layers.84.mlp.experts.109.gate_proj", "model.layers.84.mlp.experts.110.gate_proj", "model.layers.84.mlp.experts.111.gate_proj", "model.layers.84.mlp.experts.112.gate_proj", "model.layers.84.mlp.experts.113.gate_proj", "model.layers.84.mlp.experts.114.gate_proj", "model.layers.84.mlp.experts.115.gate_proj", "model.layers.84.mlp.experts.116.gate_proj", "model.layers.84.mlp.experts.117.gate_proj", "model.layers.84.mlp.experts.118.gate_proj", "model.layers.84.mlp.experts.119.gate_proj", "model.layers.84.mlp.experts.120.gate_proj", "model.layers.84.mlp.experts.121.gate_proj", "model.layers.84.mlp.experts.122.gate_proj", "model.layers.84.mlp.experts.123.gate_proj", "model.layers.84.mlp.experts.124.gate_proj", "model.layers.84.mlp.experts.125.gate_proj", "model.layers.84.mlp.experts.126.gate_proj", "model.layers.84.mlp.experts.127.gate_proj", "model.layers.84.mlp.experts.128.gate_proj", "model.layers.84.mlp.experts.129.gate_proj", "model.layers.84.mlp.experts.130.gate_proj", "model.layers.84.mlp.experts.131.gate_proj", "model.layers.84.mlp.experts.132.gate_proj", "model.layers.84.mlp.experts.133.gate_proj", "model.layers.84.mlp.experts.134.gate_proj", "model.layers.84.mlp.experts.135.gate_proj", "model.layers.84.mlp.experts.136.gate_proj", "model.layers.84.mlp.experts.137.gate_proj", "model.layers.84.mlp.experts.138.gate_proj", "model.layers.84.mlp.experts.139.gate_proj", "model.layers.84.mlp.experts.140.gate_proj", "model.layers.84.mlp.experts.141.gate_proj", "model.layers.84.mlp.experts.142.gate_proj", "model.layers.84.mlp.experts.143.gate_proj", "model.layers.84.mlp.experts.144.gate_proj", "model.layers.84.mlp.experts.145.gate_proj", "model.layers.84.mlp.experts.146.gate_proj", "model.layers.84.mlp.experts.147.gate_proj", "model.layers.84.mlp.experts.148.gate_proj", "model.layers.84.mlp.experts.149.gate_proj", "model.layers.84.mlp.experts.150.gate_proj", "model.layers.84.mlp.experts.151.gate_proj", "model.layers.84.mlp.experts.152.gate_proj", "model.layers.84.mlp.experts.153.gate_proj", "model.layers.84.mlp.experts.154.gate_proj", "model.layers.84.mlp.experts.155.gate_proj", "model.layers.84.mlp.experts.156.gate_proj", "model.layers.84.mlp.experts.157.gate_proj", "model.layers.84.mlp.experts.158.gate_proj", "model.layers.84.mlp.experts.159.gate_proj", "model.layers.84.mlp.experts.0.up_proj", "model.layers.84.mlp.experts.1.up_proj", "model.layers.84.mlp.experts.2.up_proj", "model.layers.84.mlp.experts.3.up_proj", "model.layers.84.mlp.experts.4.up_proj", "model.layers.84.mlp.experts.5.up_proj", "model.layers.84.mlp.experts.6.up_proj", "model.layers.84.mlp.experts.7.up_proj", "model.layers.84.mlp.experts.8.up_proj", "model.layers.84.mlp.experts.9.up_proj", "model.layers.84.mlp.experts.10.up_proj", "model.layers.84.mlp.experts.11.up_proj", "model.layers.84.mlp.experts.12.up_proj", "model.layers.84.mlp.experts.13.up_proj", "model.layers.84.mlp.experts.14.up_proj", "model.layers.84.mlp.experts.15.up_proj", "model.layers.84.mlp.experts.16.up_proj", "model.layers.84.mlp.experts.17.up_proj", "model.layers.84.mlp.experts.18.up_proj", "model.layers.84.mlp.experts.19.up_proj", "model.layers.84.mlp.experts.20.up_proj", "model.layers.84.mlp.experts.21.up_proj", "model.layers.84.mlp.experts.22.up_proj", "model.layers.84.mlp.experts.23.up_proj", "model.layers.84.mlp.experts.24.up_proj", "model.layers.84.mlp.experts.25.up_proj", "model.layers.84.mlp.experts.26.up_proj", "model.layers.84.mlp.experts.27.up_proj", "model.layers.84.mlp.experts.28.up_proj", "model.layers.84.mlp.experts.29.up_proj", "model.layers.84.mlp.experts.30.up_proj", "model.layers.84.mlp.experts.31.up_proj", "model.layers.84.mlp.experts.32.up_proj", "model.layers.84.mlp.experts.33.up_proj", "model.layers.84.mlp.experts.34.up_proj", "model.layers.84.mlp.experts.35.up_proj", "model.layers.84.mlp.experts.36.up_proj", "model.layers.84.mlp.experts.37.up_proj", "model.layers.84.mlp.experts.38.up_proj", "model.layers.84.mlp.experts.39.up_proj", "model.layers.84.mlp.experts.40.up_proj", "model.layers.84.mlp.experts.41.up_proj", "model.layers.84.mlp.experts.42.up_proj", "model.layers.84.mlp.experts.43.up_proj", "model.layers.84.mlp.experts.44.up_proj", "model.layers.84.mlp.experts.45.up_proj", "model.layers.84.mlp.experts.46.up_proj", "model.layers.84.mlp.experts.47.up_proj", "model.layers.84.mlp.experts.48.up_proj", "model.layers.84.mlp.experts.49.up_proj", "model.layers.84.mlp.experts.50.up_proj", "model.layers.84.mlp.experts.51.up_proj", "model.layers.84.mlp.experts.52.up_proj", "model.layers.84.mlp.experts.53.up_proj", "model.layers.84.mlp.experts.54.up_proj", "model.layers.84.mlp.experts.55.up_proj", "model.layers.84.mlp.experts.56.up_proj", "model.layers.84.mlp.experts.57.up_proj", "model.layers.84.mlp.experts.58.up_proj", "model.layers.84.mlp.experts.59.up_proj", "model.layers.84.mlp.experts.60.up_proj", "model.layers.84.mlp.experts.61.up_proj", "model.layers.84.mlp.experts.62.up_proj", "model.layers.84.mlp.experts.63.up_proj", "model.layers.84.mlp.experts.64.up_proj", "model.layers.84.mlp.experts.65.up_proj", "model.layers.84.mlp.experts.66.up_proj", "model.layers.84.mlp.experts.67.up_proj", "model.layers.84.mlp.experts.68.up_proj", "model.layers.84.mlp.experts.69.up_proj", "model.layers.84.mlp.experts.70.up_proj", "model.layers.84.mlp.experts.71.up_proj", "model.layers.84.mlp.experts.72.up_proj", "model.layers.84.mlp.experts.73.up_proj", "model.layers.84.mlp.experts.74.up_proj", "model.layers.84.mlp.experts.75.up_proj", "model.layers.84.mlp.experts.76.up_proj", "model.layers.84.mlp.experts.77.up_proj", "model.layers.84.mlp.experts.78.up_proj", "model.layers.84.mlp.experts.79.up_proj", "model.layers.84.mlp.experts.80.up_proj", "model.layers.84.mlp.experts.81.up_proj", "model.layers.84.mlp.experts.82.up_proj", "model.layers.84.mlp.experts.83.up_proj", "model.layers.84.mlp.experts.84.up_proj", "model.layers.84.mlp.experts.85.up_proj", "model.layers.84.mlp.experts.86.up_proj", "model.layers.84.mlp.experts.87.up_proj", "model.layers.84.mlp.experts.88.up_proj", "model.layers.84.mlp.experts.89.up_proj", "model.layers.84.mlp.experts.90.up_proj", "model.layers.84.mlp.experts.91.up_proj", "model.layers.84.mlp.experts.92.up_proj", "model.layers.84.mlp.experts.93.up_proj", "model.layers.84.mlp.experts.94.up_proj", "model.layers.84.mlp.experts.95.up_proj", "model.layers.84.mlp.experts.96.up_proj", "model.layers.84.mlp.experts.97.up_proj", "model.layers.84.mlp.experts.98.up_proj", "model.layers.84.mlp.experts.99.up_proj", "model.layers.84.mlp.experts.100.up_proj", "model.layers.84.mlp.experts.101.up_proj", "model.layers.84.mlp.experts.102.up_proj", "model.layers.84.mlp.experts.103.up_proj", "model.layers.84.mlp.experts.104.up_proj", "model.layers.84.mlp.experts.105.up_proj", "model.layers.84.mlp.experts.106.up_proj", "model.layers.84.mlp.experts.107.up_proj", "model.layers.84.mlp.experts.108.up_proj", "model.layers.84.mlp.experts.109.up_proj", "model.layers.84.mlp.experts.110.up_proj", "model.layers.84.mlp.experts.111.up_proj", "model.layers.84.mlp.experts.112.up_proj", "model.layers.84.mlp.experts.113.up_proj", "model.layers.84.mlp.experts.114.up_proj", "model.layers.84.mlp.experts.115.up_proj", "model.layers.84.mlp.experts.116.up_proj", "model.layers.84.mlp.experts.117.up_proj", "model.layers.84.mlp.experts.118.up_proj", "model.layers.84.mlp.experts.119.up_proj", "model.layers.84.mlp.experts.120.up_proj", "model.layers.84.mlp.experts.121.up_proj", "model.layers.84.mlp.experts.122.up_proj", "model.layers.84.mlp.experts.123.up_proj", "model.layers.84.mlp.experts.124.up_proj", "model.layers.84.mlp.experts.125.up_proj", "model.layers.84.mlp.experts.126.up_proj", "model.layers.84.mlp.experts.127.up_proj", "model.layers.84.mlp.experts.128.up_proj", "model.layers.84.mlp.experts.129.up_proj", "model.layers.84.mlp.experts.130.up_proj", "model.layers.84.mlp.experts.131.up_proj", "model.layers.84.mlp.experts.132.up_proj", "model.layers.84.mlp.experts.133.up_proj", "model.layers.84.mlp.experts.134.up_proj", "model.layers.84.mlp.experts.135.up_proj", "model.layers.84.mlp.experts.136.up_proj", "model.layers.84.mlp.experts.137.up_proj", "model.layers.84.mlp.experts.138.up_proj", "model.layers.84.mlp.experts.139.up_proj", "model.layers.84.mlp.experts.140.up_proj", "model.layers.84.mlp.experts.141.up_proj", "model.layers.84.mlp.experts.142.up_proj", "model.layers.84.mlp.experts.143.up_proj", "model.layers.84.mlp.experts.144.up_proj", "model.layers.84.mlp.experts.145.up_proj", "model.layers.84.mlp.experts.146.up_proj", "model.layers.84.mlp.experts.147.up_proj", "model.layers.84.mlp.experts.148.up_proj", "model.layers.84.mlp.experts.149.up_proj", "model.layers.84.mlp.experts.150.up_proj", "model.layers.84.mlp.experts.151.up_proj", "model.layers.84.mlp.experts.152.up_proj", "model.layers.84.mlp.experts.153.up_proj", "model.layers.84.mlp.experts.154.up_proj", "model.layers.84.mlp.experts.155.up_proj", "model.layers.84.mlp.experts.156.up_proj", "model.layers.84.mlp.experts.157.up_proj", "model.layers.84.mlp.experts.158.up_proj", "model.layers.84.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.4289107639340378e-07, "dbits": 5033164800 } ] }, { "idx": 506, "layers": [ "model.layers.84.mlp.experts.0.down_proj", "model.layers.84.mlp.experts.1.down_proj", "model.layers.84.mlp.experts.2.down_proj", "model.layers.84.mlp.experts.3.down_proj", "model.layers.84.mlp.experts.4.down_proj", "model.layers.84.mlp.experts.5.down_proj", "model.layers.84.mlp.experts.6.down_proj", "model.layers.84.mlp.experts.7.down_proj", "model.layers.84.mlp.experts.8.down_proj", "model.layers.84.mlp.experts.9.down_proj", "model.layers.84.mlp.experts.10.down_proj", "model.layers.84.mlp.experts.11.down_proj", "model.layers.84.mlp.experts.12.down_proj", "model.layers.84.mlp.experts.13.down_proj", "model.layers.84.mlp.experts.14.down_proj", "model.layers.84.mlp.experts.15.down_proj", "model.layers.84.mlp.experts.16.down_proj", "model.layers.84.mlp.experts.17.down_proj", "model.layers.84.mlp.experts.18.down_proj", "model.layers.84.mlp.experts.19.down_proj", "model.layers.84.mlp.experts.20.down_proj", "model.layers.84.mlp.experts.21.down_proj", "model.layers.84.mlp.experts.22.down_proj", "model.layers.84.mlp.experts.23.down_proj", "model.layers.84.mlp.experts.24.down_proj", "model.layers.84.mlp.experts.25.down_proj", "model.layers.84.mlp.experts.26.down_proj", "model.layers.84.mlp.experts.27.down_proj", "model.layers.84.mlp.experts.28.down_proj", "model.layers.84.mlp.experts.29.down_proj", "model.layers.84.mlp.experts.30.down_proj", "model.layers.84.mlp.experts.31.down_proj", "model.layers.84.mlp.experts.32.down_proj", "model.layers.84.mlp.experts.33.down_proj", "model.layers.84.mlp.experts.34.down_proj", "model.layers.84.mlp.experts.35.down_proj", "model.layers.84.mlp.experts.36.down_proj", "model.layers.84.mlp.experts.37.down_proj", "model.layers.84.mlp.experts.38.down_proj", "model.layers.84.mlp.experts.39.down_proj", "model.layers.84.mlp.experts.40.down_proj", "model.layers.84.mlp.experts.41.down_proj", "model.layers.84.mlp.experts.42.down_proj", "model.layers.84.mlp.experts.43.down_proj", "model.layers.84.mlp.experts.44.down_proj", "model.layers.84.mlp.experts.45.down_proj", "model.layers.84.mlp.experts.46.down_proj", "model.layers.84.mlp.experts.47.down_proj", "model.layers.84.mlp.experts.48.down_proj", "model.layers.84.mlp.experts.49.down_proj", "model.layers.84.mlp.experts.50.down_proj", "model.layers.84.mlp.experts.51.down_proj", "model.layers.84.mlp.experts.52.down_proj", "model.layers.84.mlp.experts.53.down_proj", "model.layers.84.mlp.experts.54.down_proj", "model.layers.84.mlp.experts.55.down_proj", "model.layers.84.mlp.experts.56.down_proj", "model.layers.84.mlp.experts.57.down_proj", "model.layers.84.mlp.experts.58.down_proj", "model.layers.84.mlp.experts.59.down_proj", "model.layers.84.mlp.experts.60.down_proj", "model.layers.84.mlp.experts.61.down_proj", "model.layers.84.mlp.experts.62.down_proj", "model.layers.84.mlp.experts.63.down_proj", "model.layers.84.mlp.experts.64.down_proj", "model.layers.84.mlp.experts.65.down_proj", "model.layers.84.mlp.experts.66.down_proj", "model.layers.84.mlp.experts.67.down_proj", "model.layers.84.mlp.experts.68.down_proj", "model.layers.84.mlp.experts.69.down_proj", "model.layers.84.mlp.experts.70.down_proj", "model.layers.84.mlp.experts.71.down_proj", "model.layers.84.mlp.experts.72.down_proj", "model.layers.84.mlp.experts.73.down_proj", "model.layers.84.mlp.experts.74.down_proj", "model.layers.84.mlp.experts.75.down_proj", "model.layers.84.mlp.experts.76.down_proj", "model.layers.84.mlp.experts.77.down_proj", "model.layers.84.mlp.experts.78.down_proj", "model.layers.84.mlp.experts.79.down_proj", "model.layers.84.mlp.experts.80.down_proj", "model.layers.84.mlp.experts.81.down_proj", "model.layers.84.mlp.experts.82.down_proj", "model.layers.84.mlp.experts.83.down_proj", "model.layers.84.mlp.experts.84.down_proj", "model.layers.84.mlp.experts.85.down_proj", "model.layers.84.mlp.experts.86.down_proj", "model.layers.84.mlp.experts.87.down_proj", "model.layers.84.mlp.experts.88.down_proj", "model.layers.84.mlp.experts.89.down_proj", "model.layers.84.mlp.experts.90.down_proj", "model.layers.84.mlp.experts.91.down_proj", "model.layers.84.mlp.experts.92.down_proj", "model.layers.84.mlp.experts.93.down_proj", "model.layers.84.mlp.experts.94.down_proj", "model.layers.84.mlp.experts.95.down_proj", "model.layers.84.mlp.experts.96.down_proj", "model.layers.84.mlp.experts.97.down_proj", "model.layers.84.mlp.experts.98.down_proj", "model.layers.84.mlp.experts.99.down_proj", "model.layers.84.mlp.experts.100.down_proj", "model.layers.84.mlp.experts.101.down_proj", "model.layers.84.mlp.experts.102.down_proj", "model.layers.84.mlp.experts.103.down_proj", "model.layers.84.mlp.experts.104.down_proj", "model.layers.84.mlp.experts.105.down_proj", "model.layers.84.mlp.experts.106.down_proj", "model.layers.84.mlp.experts.107.down_proj", "model.layers.84.mlp.experts.108.down_proj", "model.layers.84.mlp.experts.109.down_proj", "model.layers.84.mlp.experts.110.down_proj", "model.layers.84.mlp.experts.111.down_proj", "model.layers.84.mlp.experts.112.down_proj", "model.layers.84.mlp.experts.113.down_proj", "model.layers.84.mlp.experts.114.down_proj", "model.layers.84.mlp.experts.115.down_proj", "model.layers.84.mlp.experts.116.down_proj", "model.layers.84.mlp.experts.117.down_proj", "model.layers.84.mlp.experts.118.down_proj", "model.layers.84.mlp.experts.119.down_proj", "model.layers.84.mlp.experts.120.down_proj", "model.layers.84.mlp.experts.121.down_proj", "model.layers.84.mlp.experts.122.down_proj", "model.layers.84.mlp.experts.123.down_proj", "model.layers.84.mlp.experts.124.down_proj", "model.layers.84.mlp.experts.125.down_proj", "model.layers.84.mlp.experts.126.down_proj", "model.layers.84.mlp.experts.127.down_proj", "model.layers.84.mlp.experts.128.down_proj", "model.layers.84.mlp.experts.129.down_proj", "model.layers.84.mlp.experts.130.down_proj", "model.layers.84.mlp.experts.131.down_proj", "model.layers.84.mlp.experts.132.down_proj", "model.layers.84.mlp.experts.133.down_proj", "model.layers.84.mlp.experts.134.down_proj", "model.layers.84.mlp.experts.135.down_proj", "model.layers.84.mlp.experts.136.down_proj", "model.layers.84.mlp.experts.137.down_proj", "model.layers.84.mlp.experts.138.down_proj", "model.layers.84.mlp.experts.139.down_proj", "model.layers.84.mlp.experts.140.down_proj", "model.layers.84.mlp.experts.141.down_proj", "model.layers.84.mlp.experts.142.down_proj", "model.layers.84.mlp.experts.143.down_proj", "model.layers.84.mlp.experts.144.down_proj", "model.layers.84.mlp.experts.145.down_proj", "model.layers.84.mlp.experts.146.down_proj", "model.layers.84.mlp.experts.147.down_proj", "model.layers.84.mlp.experts.148.down_proj", "model.layers.84.mlp.experts.149.down_proj", "model.layers.84.mlp.experts.150.down_proj", "model.layers.84.mlp.experts.151.down_proj", "model.layers.84.mlp.experts.152.down_proj", "model.layers.84.mlp.experts.153.down_proj", "model.layers.84.mlp.experts.154.down_proj", "model.layers.84.mlp.experts.155.down_proj", "model.layers.84.mlp.experts.156.down_proj", "model.layers.84.mlp.experts.157.down_proj", "model.layers.84.mlp.experts.158.down_proj", "model.layers.84.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.4078472051771663e-07, "dbits": 2516582400 } ] }, { "idx": 507, "layers": [ "model.layers.85.self_attn.q_proj" ], "candidates": [ { "dkld": -3.772450145333962e-06, "dbits": 125829120 } ] }, { "idx": 508, "layers": [ "model.layers.85.self_attn.k_proj", "model.layers.85.self_attn.v_proj" ], "candidates": [ { "dkld": -8.862937102094271e-06, "dbits": 20971520 } ] }, { "idx": 509, "layers": [ "model.layers.85.self_attn.o_proj" ], "candidates": [ { "dkld": -2.218520967289827e-06, "dbits": 125829120 } ] }, { "idx": 510, "layers": [ "model.layers.85.mlp.shared_experts.gate_proj", "model.layers.85.mlp.shared_experts.up_proj", "model.layers.85.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -4.764995537697835e-07, "dbits": 47185920 } ] }, { "idx": 511, "layers": [ "model.layers.85.mlp.experts.0.gate_proj", "model.layers.85.mlp.experts.1.gate_proj", "model.layers.85.mlp.experts.2.gate_proj", "model.layers.85.mlp.experts.3.gate_proj", "model.layers.85.mlp.experts.4.gate_proj", "model.layers.85.mlp.experts.5.gate_proj", "model.layers.85.mlp.experts.6.gate_proj", "model.layers.85.mlp.experts.7.gate_proj", "model.layers.85.mlp.experts.8.gate_proj", "model.layers.85.mlp.experts.9.gate_proj", "model.layers.85.mlp.experts.10.gate_proj", "model.layers.85.mlp.experts.11.gate_proj", "model.layers.85.mlp.experts.12.gate_proj", "model.layers.85.mlp.experts.13.gate_proj", "model.layers.85.mlp.experts.14.gate_proj", "model.layers.85.mlp.experts.15.gate_proj", "model.layers.85.mlp.experts.16.gate_proj", "model.layers.85.mlp.experts.17.gate_proj", "model.layers.85.mlp.experts.18.gate_proj", "model.layers.85.mlp.experts.19.gate_proj", "model.layers.85.mlp.experts.20.gate_proj", "model.layers.85.mlp.experts.21.gate_proj", "model.layers.85.mlp.experts.22.gate_proj", "model.layers.85.mlp.experts.23.gate_proj", "model.layers.85.mlp.experts.24.gate_proj", "model.layers.85.mlp.experts.25.gate_proj", "model.layers.85.mlp.experts.26.gate_proj", "model.layers.85.mlp.experts.27.gate_proj", "model.layers.85.mlp.experts.28.gate_proj", "model.layers.85.mlp.experts.29.gate_proj", "model.layers.85.mlp.experts.30.gate_proj", "model.layers.85.mlp.experts.31.gate_proj", "model.layers.85.mlp.experts.32.gate_proj", "model.layers.85.mlp.experts.33.gate_proj", "model.layers.85.mlp.experts.34.gate_proj", "model.layers.85.mlp.experts.35.gate_proj", "model.layers.85.mlp.experts.36.gate_proj", "model.layers.85.mlp.experts.37.gate_proj", "model.layers.85.mlp.experts.38.gate_proj", "model.layers.85.mlp.experts.39.gate_proj", "model.layers.85.mlp.experts.40.gate_proj", "model.layers.85.mlp.experts.41.gate_proj", "model.layers.85.mlp.experts.42.gate_proj", "model.layers.85.mlp.experts.43.gate_proj", "model.layers.85.mlp.experts.44.gate_proj", "model.layers.85.mlp.experts.45.gate_proj", "model.layers.85.mlp.experts.46.gate_proj", "model.layers.85.mlp.experts.47.gate_proj", "model.layers.85.mlp.experts.48.gate_proj", "model.layers.85.mlp.experts.49.gate_proj", "model.layers.85.mlp.experts.50.gate_proj", "model.layers.85.mlp.experts.51.gate_proj", "model.layers.85.mlp.experts.52.gate_proj", "model.layers.85.mlp.experts.53.gate_proj", "model.layers.85.mlp.experts.54.gate_proj", "model.layers.85.mlp.experts.55.gate_proj", "model.layers.85.mlp.experts.56.gate_proj", "model.layers.85.mlp.experts.57.gate_proj", "model.layers.85.mlp.experts.58.gate_proj", "model.layers.85.mlp.experts.59.gate_proj", "model.layers.85.mlp.experts.60.gate_proj", "model.layers.85.mlp.experts.61.gate_proj", "model.layers.85.mlp.experts.62.gate_proj", "model.layers.85.mlp.experts.63.gate_proj", "model.layers.85.mlp.experts.64.gate_proj", "model.layers.85.mlp.experts.65.gate_proj", "model.layers.85.mlp.experts.66.gate_proj", "model.layers.85.mlp.experts.67.gate_proj", "model.layers.85.mlp.experts.68.gate_proj", "model.layers.85.mlp.experts.69.gate_proj", "model.layers.85.mlp.experts.70.gate_proj", "model.layers.85.mlp.experts.71.gate_proj", "model.layers.85.mlp.experts.72.gate_proj", "model.layers.85.mlp.experts.73.gate_proj", "model.layers.85.mlp.experts.74.gate_proj", "model.layers.85.mlp.experts.75.gate_proj", "model.layers.85.mlp.experts.76.gate_proj", "model.layers.85.mlp.experts.77.gate_proj", "model.layers.85.mlp.experts.78.gate_proj", "model.layers.85.mlp.experts.79.gate_proj", "model.layers.85.mlp.experts.80.gate_proj", "model.layers.85.mlp.experts.81.gate_proj", "model.layers.85.mlp.experts.82.gate_proj", "model.layers.85.mlp.experts.83.gate_proj", "model.layers.85.mlp.experts.84.gate_proj", "model.layers.85.mlp.experts.85.gate_proj", "model.layers.85.mlp.experts.86.gate_proj", "model.layers.85.mlp.experts.87.gate_proj", "model.layers.85.mlp.experts.88.gate_proj", "model.layers.85.mlp.experts.89.gate_proj", "model.layers.85.mlp.experts.90.gate_proj", "model.layers.85.mlp.experts.91.gate_proj", "model.layers.85.mlp.experts.92.gate_proj", "model.layers.85.mlp.experts.93.gate_proj", "model.layers.85.mlp.experts.94.gate_proj", "model.layers.85.mlp.experts.95.gate_proj", "model.layers.85.mlp.experts.96.gate_proj", "model.layers.85.mlp.experts.97.gate_proj", "model.layers.85.mlp.experts.98.gate_proj", "model.layers.85.mlp.experts.99.gate_proj", "model.layers.85.mlp.experts.100.gate_proj", "model.layers.85.mlp.experts.101.gate_proj", "model.layers.85.mlp.experts.102.gate_proj", "model.layers.85.mlp.experts.103.gate_proj", "model.layers.85.mlp.experts.104.gate_proj", "model.layers.85.mlp.experts.105.gate_proj", "model.layers.85.mlp.experts.106.gate_proj", "model.layers.85.mlp.experts.107.gate_proj", "model.layers.85.mlp.experts.108.gate_proj", "model.layers.85.mlp.experts.109.gate_proj", "model.layers.85.mlp.experts.110.gate_proj", "model.layers.85.mlp.experts.111.gate_proj", "model.layers.85.mlp.experts.112.gate_proj", "model.layers.85.mlp.experts.113.gate_proj", "model.layers.85.mlp.experts.114.gate_proj", "model.layers.85.mlp.experts.115.gate_proj", "model.layers.85.mlp.experts.116.gate_proj", "model.layers.85.mlp.experts.117.gate_proj", "model.layers.85.mlp.experts.118.gate_proj", "model.layers.85.mlp.experts.119.gate_proj", "model.layers.85.mlp.experts.120.gate_proj", "model.layers.85.mlp.experts.121.gate_proj", "model.layers.85.mlp.experts.122.gate_proj", "model.layers.85.mlp.experts.123.gate_proj", "model.layers.85.mlp.experts.124.gate_proj", "model.layers.85.mlp.experts.125.gate_proj", "model.layers.85.mlp.experts.126.gate_proj", "model.layers.85.mlp.experts.127.gate_proj", "model.layers.85.mlp.experts.128.gate_proj", "model.layers.85.mlp.experts.129.gate_proj", "model.layers.85.mlp.experts.130.gate_proj", "model.layers.85.mlp.experts.131.gate_proj", "model.layers.85.mlp.experts.132.gate_proj", "model.layers.85.mlp.experts.133.gate_proj", "model.layers.85.mlp.experts.134.gate_proj", "model.layers.85.mlp.experts.135.gate_proj", "model.layers.85.mlp.experts.136.gate_proj", "model.layers.85.mlp.experts.137.gate_proj", "model.layers.85.mlp.experts.138.gate_proj", "model.layers.85.mlp.experts.139.gate_proj", "model.layers.85.mlp.experts.140.gate_proj", "model.layers.85.mlp.experts.141.gate_proj", "model.layers.85.mlp.experts.142.gate_proj", "model.layers.85.mlp.experts.143.gate_proj", "model.layers.85.mlp.experts.144.gate_proj", "model.layers.85.mlp.experts.145.gate_proj", "model.layers.85.mlp.experts.146.gate_proj", "model.layers.85.mlp.experts.147.gate_proj", "model.layers.85.mlp.experts.148.gate_proj", "model.layers.85.mlp.experts.149.gate_proj", "model.layers.85.mlp.experts.150.gate_proj", "model.layers.85.mlp.experts.151.gate_proj", "model.layers.85.mlp.experts.152.gate_proj", "model.layers.85.mlp.experts.153.gate_proj", "model.layers.85.mlp.experts.154.gate_proj", "model.layers.85.mlp.experts.155.gate_proj", "model.layers.85.mlp.experts.156.gate_proj", "model.layers.85.mlp.experts.157.gate_proj", "model.layers.85.mlp.experts.158.gate_proj", "model.layers.85.mlp.experts.159.gate_proj", "model.layers.85.mlp.experts.0.up_proj", "model.layers.85.mlp.experts.1.up_proj", "model.layers.85.mlp.experts.2.up_proj", "model.layers.85.mlp.experts.3.up_proj", "model.layers.85.mlp.experts.4.up_proj", "model.layers.85.mlp.experts.5.up_proj", "model.layers.85.mlp.experts.6.up_proj", "model.layers.85.mlp.experts.7.up_proj", "model.layers.85.mlp.experts.8.up_proj", "model.layers.85.mlp.experts.9.up_proj", "model.layers.85.mlp.experts.10.up_proj", "model.layers.85.mlp.experts.11.up_proj", "model.layers.85.mlp.experts.12.up_proj", "model.layers.85.mlp.experts.13.up_proj", "model.layers.85.mlp.experts.14.up_proj", "model.layers.85.mlp.experts.15.up_proj", "model.layers.85.mlp.experts.16.up_proj", "model.layers.85.mlp.experts.17.up_proj", "model.layers.85.mlp.experts.18.up_proj", "model.layers.85.mlp.experts.19.up_proj", "model.layers.85.mlp.experts.20.up_proj", "model.layers.85.mlp.experts.21.up_proj", "model.layers.85.mlp.experts.22.up_proj", "model.layers.85.mlp.experts.23.up_proj", "model.layers.85.mlp.experts.24.up_proj", "model.layers.85.mlp.experts.25.up_proj", "model.layers.85.mlp.experts.26.up_proj", "model.layers.85.mlp.experts.27.up_proj", "model.layers.85.mlp.experts.28.up_proj", "model.layers.85.mlp.experts.29.up_proj", "model.layers.85.mlp.experts.30.up_proj", "model.layers.85.mlp.experts.31.up_proj", "model.layers.85.mlp.experts.32.up_proj", "model.layers.85.mlp.experts.33.up_proj", "model.layers.85.mlp.experts.34.up_proj", "model.layers.85.mlp.experts.35.up_proj", "model.layers.85.mlp.experts.36.up_proj", "model.layers.85.mlp.experts.37.up_proj", "model.layers.85.mlp.experts.38.up_proj", "model.layers.85.mlp.experts.39.up_proj", "model.layers.85.mlp.experts.40.up_proj", "model.layers.85.mlp.experts.41.up_proj", "model.layers.85.mlp.experts.42.up_proj", "model.layers.85.mlp.experts.43.up_proj", "model.layers.85.mlp.experts.44.up_proj", "model.layers.85.mlp.experts.45.up_proj", "model.layers.85.mlp.experts.46.up_proj", "model.layers.85.mlp.experts.47.up_proj", "model.layers.85.mlp.experts.48.up_proj", "model.layers.85.mlp.experts.49.up_proj", "model.layers.85.mlp.experts.50.up_proj", "model.layers.85.mlp.experts.51.up_proj", "model.layers.85.mlp.experts.52.up_proj", "model.layers.85.mlp.experts.53.up_proj", "model.layers.85.mlp.experts.54.up_proj", "model.layers.85.mlp.experts.55.up_proj", "model.layers.85.mlp.experts.56.up_proj", "model.layers.85.mlp.experts.57.up_proj", "model.layers.85.mlp.experts.58.up_proj", "model.layers.85.mlp.experts.59.up_proj", "model.layers.85.mlp.experts.60.up_proj", "model.layers.85.mlp.experts.61.up_proj", "model.layers.85.mlp.experts.62.up_proj", "model.layers.85.mlp.experts.63.up_proj", "model.layers.85.mlp.experts.64.up_proj", "model.layers.85.mlp.experts.65.up_proj", "model.layers.85.mlp.experts.66.up_proj", "model.layers.85.mlp.experts.67.up_proj", "model.layers.85.mlp.experts.68.up_proj", "model.layers.85.mlp.experts.69.up_proj", "model.layers.85.mlp.experts.70.up_proj", "model.layers.85.mlp.experts.71.up_proj", "model.layers.85.mlp.experts.72.up_proj", "model.layers.85.mlp.experts.73.up_proj", "model.layers.85.mlp.experts.74.up_proj", "model.layers.85.mlp.experts.75.up_proj", "model.layers.85.mlp.experts.76.up_proj", "model.layers.85.mlp.experts.77.up_proj", "model.layers.85.mlp.experts.78.up_proj", "model.layers.85.mlp.experts.79.up_proj", "model.layers.85.mlp.experts.80.up_proj", "model.layers.85.mlp.experts.81.up_proj", "model.layers.85.mlp.experts.82.up_proj", "model.layers.85.mlp.experts.83.up_proj", "model.layers.85.mlp.experts.84.up_proj", "model.layers.85.mlp.experts.85.up_proj", "model.layers.85.mlp.experts.86.up_proj", "model.layers.85.mlp.experts.87.up_proj", "model.layers.85.mlp.experts.88.up_proj", "model.layers.85.mlp.experts.89.up_proj", "model.layers.85.mlp.experts.90.up_proj", "model.layers.85.mlp.experts.91.up_proj", "model.layers.85.mlp.experts.92.up_proj", "model.layers.85.mlp.experts.93.up_proj", "model.layers.85.mlp.experts.94.up_proj", "model.layers.85.mlp.experts.95.up_proj", "model.layers.85.mlp.experts.96.up_proj", "model.layers.85.mlp.experts.97.up_proj", "model.layers.85.mlp.experts.98.up_proj", "model.layers.85.mlp.experts.99.up_proj", "model.layers.85.mlp.experts.100.up_proj", "model.layers.85.mlp.experts.101.up_proj", "model.layers.85.mlp.experts.102.up_proj", "model.layers.85.mlp.experts.103.up_proj", "model.layers.85.mlp.experts.104.up_proj", "model.layers.85.mlp.experts.105.up_proj", "model.layers.85.mlp.experts.106.up_proj", "model.layers.85.mlp.experts.107.up_proj", "model.layers.85.mlp.experts.108.up_proj", "model.layers.85.mlp.experts.109.up_proj", "model.layers.85.mlp.experts.110.up_proj", "model.layers.85.mlp.experts.111.up_proj", "model.layers.85.mlp.experts.112.up_proj", "model.layers.85.mlp.experts.113.up_proj", "model.layers.85.mlp.experts.114.up_proj", "model.layers.85.mlp.experts.115.up_proj", "model.layers.85.mlp.experts.116.up_proj", "model.layers.85.mlp.experts.117.up_proj", "model.layers.85.mlp.experts.118.up_proj", "model.layers.85.mlp.experts.119.up_proj", "model.layers.85.mlp.experts.120.up_proj", "model.layers.85.mlp.experts.121.up_proj", "model.layers.85.mlp.experts.122.up_proj", "model.layers.85.mlp.experts.123.up_proj", "model.layers.85.mlp.experts.124.up_proj", "model.layers.85.mlp.experts.125.up_proj", "model.layers.85.mlp.experts.126.up_proj", "model.layers.85.mlp.experts.127.up_proj", "model.layers.85.mlp.experts.128.up_proj", "model.layers.85.mlp.experts.129.up_proj", "model.layers.85.mlp.experts.130.up_proj", "model.layers.85.mlp.experts.131.up_proj", "model.layers.85.mlp.experts.132.up_proj", "model.layers.85.mlp.experts.133.up_proj", "model.layers.85.mlp.experts.134.up_proj", "model.layers.85.mlp.experts.135.up_proj", "model.layers.85.mlp.experts.136.up_proj", "model.layers.85.mlp.experts.137.up_proj", "model.layers.85.mlp.experts.138.up_proj", "model.layers.85.mlp.experts.139.up_proj", "model.layers.85.mlp.experts.140.up_proj", "model.layers.85.mlp.experts.141.up_proj", "model.layers.85.mlp.experts.142.up_proj", "model.layers.85.mlp.experts.143.up_proj", "model.layers.85.mlp.experts.144.up_proj", "model.layers.85.mlp.experts.145.up_proj", "model.layers.85.mlp.experts.146.up_proj", "model.layers.85.mlp.experts.147.up_proj", "model.layers.85.mlp.experts.148.up_proj", "model.layers.85.mlp.experts.149.up_proj", "model.layers.85.mlp.experts.150.up_proj", "model.layers.85.mlp.experts.151.up_proj", "model.layers.85.mlp.experts.152.up_proj", "model.layers.85.mlp.experts.153.up_proj", "model.layers.85.mlp.experts.154.up_proj", "model.layers.85.mlp.experts.155.up_proj", "model.layers.85.mlp.experts.156.up_proj", "model.layers.85.mlp.experts.157.up_proj", "model.layers.85.mlp.experts.158.up_proj", "model.layers.85.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.1314084986224337e-06, "dbits": 5033164800 } ] }, { "idx": 512, "layers": [ "model.layers.85.mlp.experts.0.down_proj", "model.layers.85.mlp.experts.1.down_proj", "model.layers.85.mlp.experts.2.down_proj", "model.layers.85.mlp.experts.3.down_proj", "model.layers.85.mlp.experts.4.down_proj", "model.layers.85.mlp.experts.5.down_proj", "model.layers.85.mlp.experts.6.down_proj", "model.layers.85.mlp.experts.7.down_proj", "model.layers.85.mlp.experts.8.down_proj", "model.layers.85.mlp.experts.9.down_proj", "model.layers.85.mlp.experts.10.down_proj", "model.layers.85.mlp.experts.11.down_proj", "model.layers.85.mlp.experts.12.down_proj", "model.layers.85.mlp.experts.13.down_proj", "model.layers.85.mlp.experts.14.down_proj", "model.layers.85.mlp.experts.15.down_proj", "model.layers.85.mlp.experts.16.down_proj", "model.layers.85.mlp.experts.17.down_proj", "model.layers.85.mlp.experts.18.down_proj", "model.layers.85.mlp.experts.19.down_proj", "model.layers.85.mlp.experts.20.down_proj", "model.layers.85.mlp.experts.21.down_proj", "model.layers.85.mlp.experts.22.down_proj", "model.layers.85.mlp.experts.23.down_proj", "model.layers.85.mlp.experts.24.down_proj", "model.layers.85.mlp.experts.25.down_proj", "model.layers.85.mlp.experts.26.down_proj", "model.layers.85.mlp.experts.27.down_proj", "model.layers.85.mlp.experts.28.down_proj", "model.layers.85.mlp.experts.29.down_proj", "model.layers.85.mlp.experts.30.down_proj", "model.layers.85.mlp.experts.31.down_proj", "model.layers.85.mlp.experts.32.down_proj", "model.layers.85.mlp.experts.33.down_proj", "model.layers.85.mlp.experts.34.down_proj", "model.layers.85.mlp.experts.35.down_proj", "model.layers.85.mlp.experts.36.down_proj", "model.layers.85.mlp.experts.37.down_proj", "model.layers.85.mlp.experts.38.down_proj", "model.layers.85.mlp.experts.39.down_proj", "model.layers.85.mlp.experts.40.down_proj", "model.layers.85.mlp.experts.41.down_proj", "model.layers.85.mlp.experts.42.down_proj", "model.layers.85.mlp.experts.43.down_proj", "model.layers.85.mlp.experts.44.down_proj", "model.layers.85.mlp.experts.45.down_proj", "model.layers.85.mlp.experts.46.down_proj", "model.layers.85.mlp.experts.47.down_proj", "model.layers.85.mlp.experts.48.down_proj", "model.layers.85.mlp.experts.49.down_proj", "model.layers.85.mlp.experts.50.down_proj", "model.layers.85.mlp.experts.51.down_proj", "model.layers.85.mlp.experts.52.down_proj", "model.layers.85.mlp.experts.53.down_proj", "model.layers.85.mlp.experts.54.down_proj", "model.layers.85.mlp.experts.55.down_proj", "model.layers.85.mlp.experts.56.down_proj", "model.layers.85.mlp.experts.57.down_proj", "model.layers.85.mlp.experts.58.down_proj", "model.layers.85.mlp.experts.59.down_proj", "model.layers.85.mlp.experts.60.down_proj", "model.layers.85.mlp.experts.61.down_proj", "model.layers.85.mlp.experts.62.down_proj", "model.layers.85.mlp.experts.63.down_proj", "model.layers.85.mlp.experts.64.down_proj", "model.layers.85.mlp.experts.65.down_proj", "model.layers.85.mlp.experts.66.down_proj", "model.layers.85.mlp.experts.67.down_proj", "model.layers.85.mlp.experts.68.down_proj", "model.layers.85.mlp.experts.69.down_proj", "model.layers.85.mlp.experts.70.down_proj", "model.layers.85.mlp.experts.71.down_proj", "model.layers.85.mlp.experts.72.down_proj", "model.layers.85.mlp.experts.73.down_proj", "model.layers.85.mlp.experts.74.down_proj", "model.layers.85.mlp.experts.75.down_proj", "model.layers.85.mlp.experts.76.down_proj", "model.layers.85.mlp.experts.77.down_proj", "model.layers.85.mlp.experts.78.down_proj", "model.layers.85.mlp.experts.79.down_proj", "model.layers.85.mlp.experts.80.down_proj", "model.layers.85.mlp.experts.81.down_proj", "model.layers.85.mlp.experts.82.down_proj", "model.layers.85.mlp.experts.83.down_proj", "model.layers.85.mlp.experts.84.down_proj", "model.layers.85.mlp.experts.85.down_proj", "model.layers.85.mlp.experts.86.down_proj", "model.layers.85.mlp.experts.87.down_proj", "model.layers.85.mlp.experts.88.down_proj", "model.layers.85.mlp.experts.89.down_proj", "model.layers.85.mlp.experts.90.down_proj", "model.layers.85.mlp.experts.91.down_proj", "model.layers.85.mlp.experts.92.down_proj", "model.layers.85.mlp.experts.93.down_proj", "model.layers.85.mlp.experts.94.down_proj", "model.layers.85.mlp.experts.95.down_proj", "model.layers.85.mlp.experts.96.down_proj", "model.layers.85.mlp.experts.97.down_proj", "model.layers.85.mlp.experts.98.down_proj", "model.layers.85.mlp.experts.99.down_proj", "model.layers.85.mlp.experts.100.down_proj", "model.layers.85.mlp.experts.101.down_proj", "model.layers.85.mlp.experts.102.down_proj", "model.layers.85.mlp.experts.103.down_proj", "model.layers.85.mlp.experts.104.down_proj", "model.layers.85.mlp.experts.105.down_proj", "model.layers.85.mlp.experts.106.down_proj", "model.layers.85.mlp.experts.107.down_proj", "model.layers.85.mlp.experts.108.down_proj", "model.layers.85.mlp.experts.109.down_proj", "model.layers.85.mlp.experts.110.down_proj", "model.layers.85.mlp.experts.111.down_proj", "model.layers.85.mlp.experts.112.down_proj", "model.layers.85.mlp.experts.113.down_proj", "model.layers.85.mlp.experts.114.down_proj", "model.layers.85.mlp.experts.115.down_proj", "model.layers.85.mlp.experts.116.down_proj", "model.layers.85.mlp.experts.117.down_proj", "model.layers.85.mlp.experts.118.down_proj", "model.layers.85.mlp.experts.119.down_proj", "model.layers.85.mlp.experts.120.down_proj", "model.layers.85.mlp.experts.121.down_proj", "model.layers.85.mlp.experts.122.down_proj", "model.layers.85.mlp.experts.123.down_proj", "model.layers.85.mlp.experts.124.down_proj", "model.layers.85.mlp.experts.125.down_proj", "model.layers.85.mlp.experts.126.down_proj", "model.layers.85.mlp.experts.127.down_proj", "model.layers.85.mlp.experts.128.down_proj", "model.layers.85.mlp.experts.129.down_proj", "model.layers.85.mlp.experts.130.down_proj", "model.layers.85.mlp.experts.131.down_proj", "model.layers.85.mlp.experts.132.down_proj", "model.layers.85.mlp.experts.133.down_proj", "model.layers.85.mlp.experts.134.down_proj", "model.layers.85.mlp.experts.135.down_proj", "model.layers.85.mlp.experts.136.down_proj", "model.layers.85.mlp.experts.137.down_proj", "model.layers.85.mlp.experts.138.down_proj", "model.layers.85.mlp.experts.139.down_proj", "model.layers.85.mlp.experts.140.down_proj", "model.layers.85.mlp.experts.141.down_proj", "model.layers.85.mlp.experts.142.down_proj", "model.layers.85.mlp.experts.143.down_proj", "model.layers.85.mlp.experts.144.down_proj", "model.layers.85.mlp.experts.145.down_proj", "model.layers.85.mlp.experts.146.down_proj", "model.layers.85.mlp.experts.147.down_proj", "model.layers.85.mlp.experts.148.down_proj", "model.layers.85.mlp.experts.149.down_proj", "model.layers.85.mlp.experts.150.down_proj", "model.layers.85.mlp.experts.151.down_proj", "model.layers.85.mlp.experts.152.down_proj", "model.layers.85.mlp.experts.153.down_proj", "model.layers.85.mlp.experts.154.down_proj", "model.layers.85.mlp.experts.155.down_proj", "model.layers.85.mlp.experts.156.down_proj", "model.layers.85.mlp.experts.157.down_proj", "model.layers.85.mlp.experts.158.down_proj", "model.layers.85.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.223555929958842e-06, "dbits": 2516582400 } ] }, { "idx": 513, "layers": [ "model.layers.86.self_attn.q_proj" ], "candidates": [ { "dkld": -3.707216819747849e-07, "dbits": 125829120 } ] }, { "idx": 514, "layers": [ "model.layers.86.self_attn.k_proj", "model.layers.86.self_attn.v_proj" ], "candidates": [ { "dkld": -5.105853779241464e-06, "dbits": 20971520 } ] }, { "idx": 515, "layers": [ "model.layers.86.self_attn.o_proj" ], "candidates": [ { "dkld": -1.213967334479137e-06, "dbits": 125829120 } ] }, { "idx": 516, "layers": [ "model.layers.86.mlp.shared_experts.gate_proj", "model.layers.86.mlp.shared_experts.up_proj", "model.layers.86.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -4.435132723301649e-06, "dbits": 47185920 } ] }, { "idx": 517, "layers": [ "model.layers.86.mlp.experts.0.gate_proj", "model.layers.86.mlp.experts.1.gate_proj", "model.layers.86.mlp.experts.2.gate_proj", "model.layers.86.mlp.experts.3.gate_proj", "model.layers.86.mlp.experts.4.gate_proj", "model.layers.86.mlp.experts.5.gate_proj", "model.layers.86.mlp.experts.6.gate_proj", "model.layers.86.mlp.experts.7.gate_proj", "model.layers.86.mlp.experts.8.gate_proj", "model.layers.86.mlp.experts.9.gate_proj", "model.layers.86.mlp.experts.10.gate_proj", "model.layers.86.mlp.experts.11.gate_proj", "model.layers.86.mlp.experts.12.gate_proj", "model.layers.86.mlp.experts.13.gate_proj", "model.layers.86.mlp.experts.14.gate_proj", "model.layers.86.mlp.experts.15.gate_proj", "model.layers.86.mlp.experts.16.gate_proj", "model.layers.86.mlp.experts.17.gate_proj", "model.layers.86.mlp.experts.18.gate_proj", "model.layers.86.mlp.experts.19.gate_proj", "model.layers.86.mlp.experts.20.gate_proj", "model.layers.86.mlp.experts.21.gate_proj", "model.layers.86.mlp.experts.22.gate_proj", "model.layers.86.mlp.experts.23.gate_proj", "model.layers.86.mlp.experts.24.gate_proj", "model.layers.86.mlp.experts.25.gate_proj", "model.layers.86.mlp.experts.26.gate_proj", "model.layers.86.mlp.experts.27.gate_proj", "model.layers.86.mlp.experts.28.gate_proj", "model.layers.86.mlp.experts.29.gate_proj", "model.layers.86.mlp.experts.30.gate_proj", "model.layers.86.mlp.experts.31.gate_proj", "model.layers.86.mlp.experts.32.gate_proj", "model.layers.86.mlp.experts.33.gate_proj", "model.layers.86.mlp.experts.34.gate_proj", "model.layers.86.mlp.experts.35.gate_proj", "model.layers.86.mlp.experts.36.gate_proj", "model.layers.86.mlp.experts.37.gate_proj", "model.layers.86.mlp.experts.38.gate_proj", "model.layers.86.mlp.experts.39.gate_proj", "model.layers.86.mlp.experts.40.gate_proj", "model.layers.86.mlp.experts.41.gate_proj", "model.layers.86.mlp.experts.42.gate_proj", "model.layers.86.mlp.experts.43.gate_proj", "model.layers.86.mlp.experts.44.gate_proj", "model.layers.86.mlp.experts.45.gate_proj", "model.layers.86.mlp.experts.46.gate_proj", "model.layers.86.mlp.experts.47.gate_proj", "model.layers.86.mlp.experts.48.gate_proj", "model.layers.86.mlp.experts.49.gate_proj", "model.layers.86.mlp.experts.50.gate_proj", "model.layers.86.mlp.experts.51.gate_proj", "model.layers.86.mlp.experts.52.gate_proj", "model.layers.86.mlp.experts.53.gate_proj", "model.layers.86.mlp.experts.54.gate_proj", "model.layers.86.mlp.experts.55.gate_proj", "model.layers.86.mlp.experts.56.gate_proj", "model.layers.86.mlp.experts.57.gate_proj", "model.layers.86.mlp.experts.58.gate_proj", "model.layers.86.mlp.experts.59.gate_proj", "model.layers.86.mlp.experts.60.gate_proj", "model.layers.86.mlp.experts.61.gate_proj", "model.layers.86.mlp.experts.62.gate_proj", "model.layers.86.mlp.experts.63.gate_proj", "model.layers.86.mlp.experts.64.gate_proj", "model.layers.86.mlp.experts.65.gate_proj", "model.layers.86.mlp.experts.66.gate_proj", "model.layers.86.mlp.experts.67.gate_proj", "model.layers.86.mlp.experts.68.gate_proj", "model.layers.86.mlp.experts.69.gate_proj", "model.layers.86.mlp.experts.70.gate_proj", "model.layers.86.mlp.experts.71.gate_proj", "model.layers.86.mlp.experts.72.gate_proj", "model.layers.86.mlp.experts.73.gate_proj", "model.layers.86.mlp.experts.74.gate_proj", "model.layers.86.mlp.experts.75.gate_proj", "model.layers.86.mlp.experts.76.gate_proj", "model.layers.86.mlp.experts.77.gate_proj", "model.layers.86.mlp.experts.78.gate_proj", "model.layers.86.mlp.experts.79.gate_proj", "model.layers.86.mlp.experts.80.gate_proj", "model.layers.86.mlp.experts.81.gate_proj", "model.layers.86.mlp.experts.82.gate_proj", "model.layers.86.mlp.experts.83.gate_proj", "model.layers.86.mlp.experts.84.gate_proj", "model.layers.86.mlp.experts.85.gate_proj", "model.layers.86.mlp.experts.86.gate_proj", "model.layers.86.mlp.experts.87.gate_proj", "model.layers.86.mlp.experts.88.gate_proj", "model.layers.86.mlp.experts.89.gate_proj", "model.layers.86.mlp.experts.90.gate_proj", "model.layers.86.mlp.experts.91.gate_proj", "model.layers.86.mlp.experts.92.gate_proj", "model.layers.86.mlp.experts.93.gate_proj", "model.layers.86.mlp.experts.94.gate_proj", "model.layers.86.mlp.experts.95.gate_proj", "model.layers.86.mlp.experts.96.gate_proj", "model.layers.86.mlp.experts.97.gate_proj", "model.layers.86.mlp.experts.98.gate_proj", "model.layers.86.mlp.experts.99.gate_proj", "model.layers.86.mlp.experts.100.gate_proj", "model.layers.86.mlp.experts.101.gate_proj", "model.layers.86.mlp.experts.102.gate_proj", "model.layers.86.mlp.experts.103.gate_proj", "model.layers.86.mlp.experts.104.gate_proj", "model.layers.86.mlp.experts.105.gate_proj", "model.layers.86.mlp.experts.106.gate_proj", "model.layers.86.mlp.experts.107.gate_proj", "model.layers.86.mlp.experts.108.gate_proj", "model.layers.86.mlp.experts.109.gate_proj", "model.layers.86.mlp.experts.110.gate_proj", "model.layers.86.mlp.experts.111.gate_proj", "model.layers.86.mlp.experts.112.gate_proj", "model.layers.86.mlp.experts.113.gate_proj", "model.layers.86.mlp.experts.114.gate_proj", "model.layers.86.mlp.experts.115.gate_proj", "model.layers.86.mlp.experts.116.gate_proj", "model.layers.86.mlp.experts.117.gate_proj", "model.layers.86.mlp.experts.118.gate_proj", "model.layers.86.mlp.experts.119.gate_proj", "model.layers.86.mlp.experts.120.gate_proj", "model.layers.86.mlp.experts.121.gate_proj", "model.layers.86.mlp.experts.122.gate_proj", "model.layers.86.mlp.experts.123.gate_proj", "model.layers.86.mlp.experts.124.gate_proj", "model.layers.86.mlp.experts.125.gate_proj", "model.layers.86.mlp.experts.126.gate_proj", "model.layers.86.mlp.experts.127.gate_proj", "model.layers.86.mlp.experts.128.gate_proj", "model.layers.86.mlp.experts.129.gate_proj", "model.layers.86.mlp.experts.130.gate_proj", "model.layers.86.mlp.experts.131.gate_proj", "model.layers.86.mlp.experts.132.gate_proj", "model.layers.86.mlp.experts.133.gate_proj", "model.layers.86.mlp.experts.134.gate_proj", "model.layers.86.mlp.experts.135.gate_proj", "model.layers.86.mlp.experts.136.gate_proj", "model.layers.86.mlp.experts.137.gate_proj", "model.layers.86.mlp.experts.138.gate_proj", "model.layers.86.mlp.experts.139.gate_proj", "model.layers.86.mlp.experts.140.gate_proj", "model.layers.86.mlp.experts.141.gate_proj", "model.layers.86.mlp.experts.142.gate_proj", "model.layers.86.mlp.experts.143.gate_proj", "model.layers.86.mlp.experts.144.gate_proj", "model.layers.86.mlp.experts.145.gate_proj", "model.layers.86.mlp.experts.146.gate_proj", "model.layers.86.mlp.experts.147.gate_proj", "model.layers.86.mlp.experts.148.gate_proj", "model.layers.86.mlp.experts.149.gate_proj", "model.layers.86.mlp.experts.150.gate_proj", "model.layers.86.mlp.experts.151.gate_proj", "model.layers.86.mlp.experts.152.gate_proj", "model.layers.86.mlp.experts.153.gate_proj", "model.layers.86.mlp.experts.154.gate_proj", "model.layers.86.mlp.experts.155.gate_proj", "model.layers.86.mlp.experts.156.gate_proj", "model.layers.86.mlp.experts.157.gate_proj", "model.layers.86.mlp.experts.158.gate_proj", "model.layers.86.mlp.experts.159.gate_proj", "model.layers.86.mlp.experts.0.up_proj", "model.layers.86.mlp.experts.1.up_proj", "model.layers.86.mlp.experts.2.up_proj", "model.layers.86.mlp.experts.3.up_proj", "model.layers.86.mlp.experts.4.up_proj", "model.layers.86.mlp.experts.5.up_proj", "model.layers.86.mlp.experts.6.up_proj", "model.layers.86.mlp.experts.7.up_proj", "model.layers.86.mlp.experts.8.up_proj", "model.layers.86.mlp.experts.9.up_proj", "model.layers.86.mlp.experts.10.up_proj", "model.layers.86.mlp.experts.11.up_proj", "model.layers.86.mlp.experts.12.up_proj", "model.layers.86.mlp.experts.13.up_proj", "model.layers.86.mlp.experts.14.up_proj", "model.layers.86.mlp.experts.15.up_proj", "model.layers.86.mlp.experts.16.up_proj", "model.layers.86.mlp.experts.17.up_proj", "model.layers.86.mlp.experts.18.up_proj", "model.layers.86.mlp.experts.19.up_proj", "model.layers.86.mlp.experts.20.up_proj", "model.layers.86.mlp.experts.21.up_proj", "model.layers.86.mlp.experts.22.up_proj", "model.layers.86.mlp.experts.23.up_proj", "model.layers.86.mlp.experts.24.up_proj", "model.layers.86.mlp.experts.25.up_proj", "model.layers.86.mlp.experts.26.up_proj", "model.layers.86.mlp.experts.27.up_proj", "model.layers.86.mlp.experts.28.up_proj", "model.layers.86.mlp.experts.29.up_proj", "model.layers.86.mlp.experts.30.up_proj", "model.layers.86.mlp.experts.31.up_proj", "model.layers.86.mlp.experts.32.up_proj", "model.layers.86.mlp.experts.33.up_proj", "model.layers.86.mlp.experts.34.up_proj", "model.layers.86.mlp.experts.35.up_proj", "model.layers.86.mlp.experts.36.up_proj", "model.layers.86.mlp.experts.37.up_proj", "model.layers.86.mlp.experts.38.up_proj", "model.layers.86.mlp.experts.39.up_proj", "model.layers.86.mlp.experts.40.up_proj", "model.layers.86.mlp.experts.41.up_proj", "model.layers.86.mlp.experts.42.up_proj", "model.layers.86.mlp.experts.43.up_proj", "model.layers.86.mlp.experts.44.up_proj", "model.layers.86.mlp.experts.45.up_proj", "model.layers.86.mlp.experts.46.up_proj", "model.layers.86.mlp.experts.47.up_proj", "model.layers.86.mlp.experts.48.up_proj", "model.layers.86.mlp.experts.49.up_proj", "model.layers.86.mlp.experts.50.up_proj", "model.layers.86.mlp.experts.51.up_proj", "model.layers.86.mlp.experts.52.up_proj", "model.layers.86.mlp.experts.53.up_proj", "model.layers.86.mlp.experts.54.up_proj", "model.layers.86.mlp.experts.55.up_proj", "model.layers.86.mlp.experts.56.up_proj", "model.layers.86.mlp.experts.57.up_proj", "model.layers.86.mlp.experts.58.up_proj", "model.layers.86.mlp.experts.59.up_proj", "model.layers.86.mlp.experts.60.up_proj", "model.layers.86.mlp.experts.61.up_proj", "model.layers.86.mlp.experts.62.up_proj", "model.layers.86.mlp.experts.63.up_proj", "model.layers.86.mlp.experts.64.up_proj", "model.layers.86.mlp.experts.65.up_proj", "model.layers.86.mlp.experts.66.up_proj", "model.layers.86.mlp.experts.67.up_proj", "model.layers.86.mlp.experts.68.up_proj", "model.layers.86.mlp.experts.69.up_proj", "model.layers.86.mlp.experts.70.up_proj", "model.layers.86.mlp.experts.71.up_proj", "model.layers.86.mlp.experts.72.up_proj", "model.layers.86.mlp.experts.73.up_proj", "model.layers.86.mlp.experts.74.up_proj", "model.layers.86.mlp.experts.75.up_proj", "model.layers.86.mlp.experts.76.up_proj", "model.layers.86.mlp.experts.77.up_proj", "model.layers.86.mlp.experts.78.up_proj", "model.layers.86.mlp.experts.79.up_proj", "model.layers.86.mlp.experts.80.up_proj", "model.layers.86.mlp.experts.81.up_proj", "model.layers.86.mlp.experts.82.up_proj", "model.layers.86.mlp.experts.83.up_proj", "model.layers.86.mlp.experts.84.up_proj", "model.layers.86.mlp.experts.85.up_proj", "model.layers.86.mlp.experts.86.up_proj", "model.layers.86.mlp.experts.87.up_proj", "model.layers.86.mlp.experts.88.up_proj", "model.layers.86.mlp.experts.89.up_proj", "model.layers.86.mlp.experts.90.up_proj", "model.layers.86.mlp.experts.91.up_proj", "model.layers.86.mlp.experts.92.up_proj", "model.layers.86.mlp.experts.93.up_proj", "model.layers.86.mlp.experts.94.up_proj", "model.layers.86.mlp.experts.95.up_proj", "model.layers.86.mlp.experts.96.up_proj", "model.layers.86.mlp.experts.97.up_proj", "model.layers.86.mlp.experts.98.up_proj", "model.layers.86.mlp.experts.99.up_proj", "model.layers.86.mlp.experts.100.up_proj", "model.layers.86.mlp.experts.101.up_proj", "model.layers.86.mlp.experts.102.up_proj", "model.layers.86.mlp.experts.103.up_proj", "model.layers.86.mlp.experts.104.up_proj", "model.layers.86.mlp.experts.105.up_proj", "model.layers.86.mlp.experts.106.up_proj", "model.layers.86.mlp.experts.107.up_proj", "model.layers.86.mlp.experts.108.up_proj", "model.layers.86.mlp.experts.109.up_proj", "model.layers.86.mlp.experts.110.up_proj", "model.layers.86.mlp.experts.111.up_proj", "model.layers.86.mlp.experts.112.up_proj", "model.layers.86.mlp.experts.113.up_proj", "model.layers.86.mlp.experts.114.up_proj", "model.layers.86.mlp.experts.115.up_proj", "model.layers.86.mlp.experts.116.up_proj", "model.layers.86.mlp.experts.117.up_proj", "model.layers.86.mlp.experts.118.up_proj", "model.layers.86.mlp.experts.119.up_proj", "model.layers.86.mlp.experts.120.up_proj", "model.layers.86.mlp.experts.121.up_proj", "model.layers.86.mlp.experts.122.up_proj", "model.layers.86.mlp.experts.123.up_proj", "model.layers.86.mlp.experts.124.up_proj", "model.layers.86.mlp.experts.125.up_proj", "model.layers.86.mlp.experts.126.up_proj", "model.layers.86.mlp.experts.127.up_proj", "model.layers.86.mlp.experts.128.up_proj", "model.layers.86.mlp.experts.129.up_proj", "model.layers.86.mlp.experts.130.up_proj", "model.layers.86.mlp.experts.131.up_proj", "model.layers.86.mlp.experts.132.up_proj", "model.layers.86.mlp.experts.133.up_proj", "model.layers.86.mlp.experts.134.up_proj", "model.layers.86.mlp.experts.135.up_proj", "model.layers.86.mlp.experts.136.up_proj", "model.layers.86.mlp.experts.137.up_proj", "model.layers.86.mlp.experts.138.up_proj", "model.layers.86.mlp.experts.139.up_proj", "model.layers.86.mlp.experts.140.up_proj", "model.layers.86.mlp.experts.141.up_proj", "model.layers.86.mlp.experts.142.up_proj", "model.layers.86.mlp.experts.143.up_proj", "model.layers.86.mlp.experts.144.up_proj", "model.layers.86.mlp.experts.145.up_proj", "model.layers.86.mlp.experts.146.up_proj", "model.layers.86.mlp.experts.147.up_proj", "model.layers.86.mlp.experts.148.up_proj", "model.layers.86.mlp.experts.149.up_proj", "model.layers.86.mlp.experts.150.up_proj", "model.layers.86.mlp.experts.151.up_proj", "model.layers.86.mlp.experts.152.up_proj", "model.layers.86.mlp.experts.153.up_proj", "model.layers.86.mlp.experts.154.up_proj", "model.layers.86.mlp.experts.155.up_proj", "model.layers.86.mlp.experts.156.up_proj", "model.layers.86.mlp.experts.157.up_proj", "model.layers.86.mlp.experts.158.up_proj", "model.layers.86.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.6751455152407505e-06, "dbits": 5033164800 } ] }, { "idx": 518, "layers": [ "model.layers.86.mlp.experts.0.down_proj", "model.layers.86.mlp.experts.1.down_proj", "model.layers.86.mlp.experts.2.down_proj", "model.layers.86.mlp.experts.3.down_proj", "model.layers.86.mlp.experts.4.down_proj", "model.layers.86.mlp.experts.5.down_proj", "model.layers.86.mlp.experts.6.down_proj", "model.layers.86.mlp.experts.7.down_proj", "model.layers.86.mlp.experts.8.down_proj", "model.layers.86.mlp.experts.9.down_proj", "model.layers.86.mlp.experts.10.down_proj", "model.layers.86.mlp.experts.11.down_proj", "model.layers.86.mlp.experts.12.down_proj", "model.layers.86.mlp.experts.13.down_proj", "model.layers.86.mlp.experts.14.down_proj", "model.layers.86.mlp.experts.15.down_proj", "model.layers.86.mlp.experts.16.down_proj", "model.layers.86.mlp.experts.17.down_proj", "model.layers.86.mlp.experts.18.down_proj", "model.layers.86.mlp.experts.19.down_proj", "model.layers.86.mlp.experts.20.down_proj", "model.layers.86.mlp.experts.21.down_proj", "model.layers.86.mlp.experts.22.down_proj", "model.layers.86.mlp.experts.23.down_proj", "model.layers.86.mlp.experts.24.down_proj", "model.layers.86.mlp.experts.25.down_proj", "model.layers.86.mlp.experts.26.down_proj", "model.layers.86.mlp.experts.27.down_proj", "model.layers.86.mlp.experts.28.down_proj", "model.layers.86.mlp.experts.29.down_proj", "model.layers.86.mlp.experts.30.down_proj", "model.layers.86.mlp.experts.31.down_proj", "model.layers.86.mlp.experts.32.down_proj", "model.layers.86.mlp.experts.33.down_proj", "model.layers.86.mlp.experts.34.down_proj", "model.layers.86.mlp.experts.35.down_proj", "model.layers.86.mlp.experts.36.down_proj", "model.layers.86.mlp.experts.37.down_proj", "model.layers.86.mlp.experts.38.down_proj", "model.layers.86.mlp.experts.39.down_proj", "model.layers.86.mlp.experts.40.down_proj", "model.layers.86.mlp.experts.41.down_proj", "model.layers.86.mlp.experts.42.down_proj", "model.layers.86.mlp.experts.43.down_proj", "model.layers.86.mlp.experts.44.down_proj", "model.layers.86.mlp.experts.45.down_proj", "model.layers.86.mlp.experts.46.down_proj", "model.layers.86.mlp.experts.47.down_proj", "model.layers.86.mlp.experts.48.down_proj", "model.layers.86.mlp.experts.49.down_proj", "model.layers.86.mlp.experts.50.down_proj", "model.layers.86.mlp.experts.51.down_proj", "model.layers.86.mlp.experts.52.down_proj", "model.layers.86.mlp.experts.53.down_proj", "model.layers.86.mlp.experts.54.down_proj", "model.layers.86.mlp.experts.55.down_proj", "model.layers.86.mlp.experts.56.down_proj", "model.layers.86.mlp.experts.57.down_proj", "model.layers.86.mlp.experts.58.down_proj", "model.layers.86.mlp.experts.59.down_proj", "model.layers.86.mlp.experts.60.down_proj", "model.layers.86.mlp.experts.61.down_proj", "model.layers.86.mlp.experts.62.down_proj", "model.layers.86.mlp.experts.63.down_proj", "model.layers.86.mlp.experts.64.down_proj", "model.layers.86.mlp.experts.65.down_proj", "model.layers.86.mlp.experts.66.down_proj", "model.layers.86.mlp.experts.67.down_proj", "model.layers.86.mlp.experts.68.down_proj", "model.layers.86.mlp.experts.69.down_proj", "model.layers.86.mlp.experts.70.down_proj", "model.layers.86.mlp.experts.71.down_proj", "model.layers.86.mlp.experts.72.down_proj", "model.layers.86.mlp.experts.73.down_proj", "model.layers.86.mlp.experts.74.down_proj", "model.layers.86.mlp.experts.75.down_proj", "model.layers.86.mlp.experts.76.down_proj", "model.layers.86.mlp.experts.77.down_proj", "model.layers.86.mlp.experts.78.down_proj", "model.layers.86.mlp.experts.79.down_proj", "model.layers.86.mlp.experts.80.down_proj", "model.layers.86.mlp.experts.81.down_proj", "model.layers.86.mlp.experts.82.down_proj", "model.layers.86.mlp.experts.83.down_proj", "model.layers.86.mlp.experts.84.down_proj", "model.layers.86.mlp.experts.85.down_proj", "model.layers.86.mlp.experts.86.down_proj", "model.layers.86.mlp.experts.87.down_proj", "model.layers.86.mlp.experts.88.down_proj", "model.layers.86.mlp.experts.89.down_proj", "model.layers.86.mlp.experts.90.down_proj", "model.layers.86.mlp.experts.91.down_proj", "model.layers.86.mlp.experts.92.down_proj", "model.layers.86.mlp.experts.93.down_proj", "model.layers.86.mlp.experts.94.down_proj", "model.layers.86.mlp.experts.95.down_proj", "model.layers.86.mlp.experts.96.down_proj", "model.layers.86.mlp.experts.97.down_proj", "model.layers.86.mlp.experts.98.down_proj", "model.layers.86.mlp.experts.99.down_proj", "model.layers.86.mlp.experts.100.down_proj", "model.layers.86.mlp.experts.101.down_proj", "model.layers.86.mlp.experts.102.down_proj", "model.layers.86.mlp.experts.103.down_proj", "model.layers.86.mlp.experts.104.down_proj", "model.layers.86.mlp.experts.105.down_proj", "model.layers.86.mlp.experts.106.down_proj", "model.layers.86.mlp.experts.107.down_proj", "model.layers.86.mlp.experts.108.down_proj", "model.layers.86.mlp.experts.109.down_proj", "model.layers.86.mlp.experts.110.down_proj", "model.layers.86.mlp.experts.111.down_proj", "model.layers.86.mlp.experts.112.down_proj", "model.layers.86.mlp.experts.113.down_proj", "model.layers.86.mlp.experts.114.down_proj", "model.layers.86.mlp.experts.115.down_proj", "model.layers.86.mlp.experts.116.down_proj", "model.layers.86.mlp.experts.117.down_proj", "model.layers.86.mlp.experts.118.down_proj", "model.layers.86.mlp.experts.119.down_proj", "model.layers.86.mlp.experts.120.down_proj", "model.layers.86.mlp.experts.121.down_proj", "model.layers.86.mlp.experts.122.down_proj", "model.layers.86.mlp.experts.123.down_proj", "model.layers.86.mlp.experts.124.down_proj", "model.layers.86.mlp.experts.125.down_proj", "model.layers.86.mlp.experts.126.down_proj", "model.layers.86.mlp.experts.127.down_proj", "model.layers.86.mlp.experts.128.down_proj", "model.layers.86.mlp.experts.129.down_proj", "model.layers.86.mlp.experts.130.down_proj", "model.layers.86.mlp.experts.131.down_proj", "model.layers.86.mlp.experts.132.down_proj", "model.layers.86.mlp.experts.133.down_proj", "model.layers.86.mlp.experts.134.down_proj", "model.layers.86.mlp.experts.135.down_proj", "model.layers.86.mlp.experts.136.down_proj", "model.layers.86.mlp.experts.137.down_proj", "model.layers.86.mlp.experts.138.down_proj", "model.layers.86.mlp.experts.139.down_proj", "model.layers.86.mlp.experts.140.down_proj", "model.layers.86.mlp.experts.141.down_proj", "model.layers.86.mlp.experts.142.down_proj", "model.layers.86.mlp.experts.143.down_proj", "model.layers.86.mlp.experts.144.down_proj", "model.layers.86.mlp.experts.145.down_proj", "model.layers.86.mlp.experts.146.down_proj", "model.layers.86.mlp.experts.147.down_proj", "model.layers.86.mlp.experts.148.down_proj", "model.layers.86.mlp.experts.149.down_proj", "model.layers.86.mlp.experts.150.down_proj", "model.layers.86.mlp.experts.151.down_proj", "model.layers.86.mlp.experts.152.down_proj", "model.layers.86.mlp.experts.153.down_proj", "model.layers.86.mlp.experts.154.down_proj", "model.layers.86.mlp.experts.155.down_proj", "model.layers.86.mlp.experts.156.down_proj", "model.layers.86.mlp.experts.157.down_proj", "model.layers.86.mlp.experts.158.down_proj", "model.layers.86.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.372410381212787e-06, "dbits": 2516582400 } ] }, { "idx": 519, "layers": [ "model.layers.87.self_attn.q_proj" ], "candidates": [ { "dkld": 6.246060365811207e-07, "dbits": 125829120 } ] }, { "idx": 520, "layers": [ "model.layers.87.self_attn.k_proj", "model.layers.87.self_attn.v_proj" ], "candidates": [ { "dkld": 3.890629159287052e-07, "dbits": 20971520 } ] }, { "idx": 521, "layers": [ "model.layers.87.self_attn.o_proj" ], "candidates": [ { "dkld": -2.035824581980662e-06, "dbits": 125829120 } ] }, { "idx": 522, "layers": [ "model.layers.87.mlp.shared_experts.gate_proj", "model.layers.87.mlp.shared_experts.up_proj", "model.layers.87.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 1.0493269655853944e-06, "dbits": 47185920 } ] }, { "idx": 523, "layers": [ "model.layers.87.mlp.experts.0.gate_proj", "model.layers.87.mlp.experts.1.gate_proj", "model.layers.87.mlp.experts.2.gate_proj", "model.layers.87.mlp.experts.3.gate_proj", "model.layers.87.mlp.experts.4.gate_proj", "model.layers.87.mlp.experts.5.gate_proj", "model.layers.87.mlp.experts.6.gate_proj", "model.layers.87.mlp.experts.7.gate_proj", "model.layers.87.mlp.experts.8.gate_proj", "model.layers.87.mlp.experts.9.gate_proj", "model.layers.87.mlp.experts.10.gate_proj", "model.layers.87.mlp.experts.11.gate_proj", "model.layers.87.mlp.experts.12.gate_proj", "model.layers.87.mlp.experts.13.gate_proj", "model.layers.87.mlp.experts.14.gate_proj", "model.layers.87.mlp.experts.15.gate_proj", "model.layers.87.mlp.experts.16.gate_proj", "model.layers.87.mlp.experts.17.gate_proj", "model.layers.87.mlp.experts.18.gate_proj", "model.layers.87.mlp.experts.19.gate_proj", "model.layers.87.mlp.experts.20.gate_proj", "model.layers.87.mlp.experts.21.gate_proj", "model.layers.87.mlp.experts.22.gate_proj", "model.layers.87.mlp.experts.23.gate_proj", "model.layers.87.mlp.experts.24.gate_proj", "model.layers.87.mlp.experts.25.gate_proj", "model.layers.87.mlp.experts.26.gate_proj", "model.layers.87.mlp.experts.27.gate_proj", "model.layers.87.mlp.experts.28.gate_proj", "model.layers.87.mlp.experts.29.gate_proj", "model.layers.87.mlp.experts.30.gate_proj", "model.layers.87.mlp.experts.31.gate_proj", "model.layers.87.mlp.experts.32.gate_proj", "model.layers.87.mlp.experts.33.gate_proj", "model.layers.87.mlp.experts.34.gate_proj", "model.layers.87.mlp.experts.35.gate_proj", "model.layers.87.mlp.experts.36.gate_proj", "model.layers.87.mlp.experts.37.gate_proj", "model.layers.87.mlp.experts.38.gate_proj", "model.layers.87.mlp.experts.39.gate_proj", "model.layers.87.mlp.experts.40.gate_proj", "model.layers.87.mlp.experts.41.gate_proj", "model.layers.87.mlp.experts.42.gate_proj", "model.layers.87.mlp.experts.43.gate_proj", "model.layers.87.mlp.experts.44.gate_proj", "model.layers.87.mlp.experts.45.gate_proj", "model.layers.87.mlp.experts.46.gate_proj", "model.layers.87.mlp.experts.47.gate_proj", "model.layers.87.mlp.experts.48.gate_proj", "model.layers.87.mlp.experts.49.gate_proj", "model.layers.87.mlp.experts.50.gate_proj", "model.layers.87.mlp.experts.51.gate_proj", "model.layers.87.mlp.experts.52.gate_proj", "model.layers.87.mlp.experts.53.gate_proj", "model.layers.87.mlp.experts.54.gate_proj", "model.layers.87.mlp.experts.55.gate_proj", "model.layers.87.mlp.experts.56.gate_proj", "model.layers.87.mlp.experts.57.gate_proj", "model.layers.87.mlp.experts.58.gate_proj", "model.layers.87.mlp.experts.59.gate_proj", "model.layers.87.mlp.experts.60.gate_proj", "model.layers.87.mlp.experts.61.gate_proj", "model.layers.87.mlp.experts.62.gate_proj", "model.layers.87.mlp.experts.63.gate_proj", "model.layers.87.mlp.experts.64.gate_proj", "model.layers.87.mlp.experts.65.gate_proj", "model.layers.87.mlp.experts.66.gate_proj", "model.layers.87.mlp.experts.67.gate_proj", "model.layers.87.mlp.experts.68.gate_proj", "model.layers.87.mlp.experts.69.gate_proj", "model.layers.87.mlp.experts.70.gate_proj", "model.layers.87.mlp.experts.71.gate_proj", "model.layers.87.mlp.experts.72.gate_proj", "model.layers.87.mlp.experts.73.gate_proj", "model.layers.87.mlp.experts.74.gate_proj", "model.layers.87.mlp.experts.75.gate_proj", "model.layers.87.mlp.experts.76.gate_proj", "model.layers.87.mlp.experts.77.gate_proj", "model.layers.87.mlp.experts.78.gate_proj", "model.layers.87.mlp.experts.79.gate_proj", "model.layers.87.mlp.experts.80.gate_proj", "model.layers.87.mlp.experts.81.gate_proj", "model.layers.87.mlp.experts.82.gate_proj", "model.layers.87.mlp.experts.83.gate_proj", "model.layers.87.mlp.experts.84.gate_proj", "model.layers.87.mlp.experts.85.gate_proj", "model.layers.87.mlp.experts.86.gate_proj", "model.layers.87.mlp.experts.87.gate_proj", "model.layers.87.mlp.experts.88.gate_proj", "model.layers.87.mlp.experts.89.gate_proj", "model.layers.87.mlp.experts.90.gate_proj", "model.layers.87.mlp.experts.91.gate_proj", "model.layers.87.mlp.experts.92.gate_proj", "model.layers.87.mlp.experts.93.gate_proj", "model.layers.87.mlp.experts.94.gate_proj", "model.layers.87.mlp.experts.95.gate_proj", "model.layers.87.mlp.experts.96.gate_proj", "model.layers.87.mlp.experts.97.gate_proj", "model.layers.87.mlp.experts.98.gate_proj", "model.layers.87.mlp.experts.99.gate_proj", "model.layers.87.mlp.experts.100.gate_proj", "model.layers.87.mlp.experts.101.gate_proj", "model.layers.87.mlp.experts.102.gate_proj", "model.layers.87.mlp.experts.103.gate_proj", "model.layers.87.mlp.experts.104.gate_proj", "model.layers.87.mlp.experts.105.gate_proj", "model.layers.87.mlp.experts.106.gate_proj", "model.layers.87.mlp.experts.107.gate_proj", "model.layers.87.mlp.experts.108.gate_proj", "model.layers.87.mlp.experts.109.gate_proj", "model.layers.87.mlp.experts.110.gate_proj", "model.layers.87.mlp.experts.111.gate_proj", "model.layers.87.mlp.experts.112.gate_proj", "model.layers.87.mlp.experts.113.gate_proj", "model.layers.87.mlp.experts.114.gate_proj", "model.layers.87.mlp.experts.115.gate_proj", "model.layers.87.mlp.experts.116.gate_proj", "model.layers.87.mlp.experts.117.gate_proj", "model.layers.87.mlp.experts.118.gate_proj", "model.layers.87.mlp.experts.119.gate_proj", "model.layers.87.mlp.experts.120.gate_proj", "model.layers.87.mlp.experts.121.gate_proj", "model.layers.87.mlp.experts.122.gate_proj", "model.layers.87.mlp.experts.123.gate_proj", "model.layers.87.mlp.experts.124.gate_proj", "model.layers.87.mlp.experts.125.gate_proj", "model.layers.87.mlp.experts.126.gate_proj", "model.layers.87.mlp.experts.127.gate_proj", "model.layers.87.mlp.experts.128.gate_proj", "model.layers.87.mlp.experts.129.gate_proj", "model.layers.87.mlp.experts.130.gate_proj", "model.layers.87.mlp.experts.131.gate_proj", "model.layers.87.mlp.experts.132.gate_proj", "model.layers.87.mlp.experts.133.gate_proj", "model.layers.87.mlp.experts.134.gate_proj", "model.layers.87.mlp.experts.135.gate_proj", "model.layers.87.mlp.experts.136.gate_proj", "model.layers.87.mlp.experts.137.gate_proj", "model.layers.87.mlp.experts.138.gate_proj", "model.layers.87.mlp.experts.139.gate_proj", "model.layers.87.mlp.experts.140.gate_proj", "model.layers.87.mlp.experts.141.gate_proj", "model.layers.87.mlp.experts.142.gate_proj", "model.layers.87.mlp.experts.143.gate_proj", "model.layers.87.mlp.experts.144.gate_proj", "model.layers.87.mlp.experts.145.gate_proj", "model.layers.87.mlp.experts.146.gate_proj", "model.layers.87.mlp.experts.147.gate_proj", "model.layers.87.mlp.experts.148.gate_proj", "model.layers.87.mlp.experts.149.gate_proj", "model.layers.87.mlp.experts.150.gate_proj", "model.layers.87.mlp.experts.151.gate_proj", "model.layers.87.mlp.experts.152.gate_proj", "model.layers.87.mlp.experts.153.gate_proj", "model.layers.87.mlp.experts.154.gate_proj", "model.layers.87.mlp.experts.155.gate_proj", "model.layers.87.mlp.experts.156.gate_proj", "model.layers.87.mlp.experts.157.gate_proj", "model.layers.87.mlp.experts.158.gate_proj", "model.layers.87.mlp.experts.159.gate_proj", "model.layers.87.mlp.experts.0.up_proj", "model.layers.87.mlp.experts.1.up_proj", "model.layers.87.mlp.experts.2.up_proj", "model.layers.87.mlp.experts.3.up_proj", "model.layers.87.mlp.experts.4.up_proj", "model.layers.87.mlp.experts.5.up_proj", "model.layers.87.mlp.experts.6.up_proj", "model.layers.87.mlp.experts.7.up_proj", "model.layers.87.mlp.experts.8.up_proj", "model.layers.87.mlp.experts.9.up_proj", "model.layers.87.mlp.experts.10.up_proj", "model.layers.87.mlp.experts.11.up_proj", "model.layers.87.mlp.experts.12.up_proj", "model.layers.87.mlp.experts.13.up_proj", "model.layers.87.mlp.experts.14.up_proj", "model.layers.87.mlp.experts.15.up_proj", "model.layers.87.mlp.experts.16.up_proj", "model.layers.87.mlp.experts.17.up_proj", "model.layers.87.mlp.experts.18.up_proj", "model.layers.87.mlp.experts.19.up_proj", "model.layers.87.mlp.experts.20.up_proj", "model.layers.87.mlp.experts.21.up_proj", "model.layers.87.mlp.experts.22.up_proj", "model.layers.87.mlp.experts.23.up_proj", "model.layers.87.mlp.experts.24.up_proj", "model.layers.87.mlp.experts.25.up_proj", "model.layers.87.mlp.experts.26.up_proj", "model.layers.87.mlp.experts.27.up_proj", "model.layers.87.mlp.experts.28.up_proj", "model.layers.87.mlp.experts.29.up_proj", "model.layers.87.mlp.experts.30.up_proj", "model.layers.87.mlp.experts.31.up_proj", "model.layers.87.mlp.experts.32.up_proj", "model.layers.87.mlp.experts.33.up_proj", "model.layers.87.mlp.experts.34.up_proj", "model.layers.87.mlp.experts.35.up_proj", "model.layers.87.mlp.experts.36.up_proj", "model.layers.87.mlp.experts.37.up_proj", "model.layers.87.mlp.experts.38.up_proj", "model.layers.87.mlp.experts.39.up_proj", "model.layers.87.mlp.experts.40.up_proj", "model.layers.87.mlp.experts.41.up_proj", "model.layers.87.mlp.experts.42.up_proj", "model.layers.87.mlp.experts.43.up_proj", "model.layers.87.mlp.experts.44.up_proj", "model.layers.87.mlp.experts.45.up_proj", "model.layers.87.mlp.experts.46.up_proj", "model.layers.87.mlp.experts.47.up_proj", "model.layers.87.mlp.experts.48.up_proj", "model.layers.87.mlp.experts.49.up_proj", "model.layers.87.mlp.experts.50.up_proj", "model.layers.87.mlp.experts.51.up_proj", "model.layers.87.mlp.experts.52.up_proj", "model.layers.87.mlp.experts.53.up_proj", "model.layers.87.mlp.experts.54.up_proj", "model.layers.87.mlp.experts.55.up_proj", "model.layers.87.mlp.experts.56.up_proj", "model.layers.87.mlp.experts.57.up_proj", "model.layers.87.mlp.experts.58.up_proj", "model.layers.87.mlp.experts.59.up_proj", "model.layers.87.mlp.experts.60.up_proj", "model.layers.87.mlp.experts.61.up_proj", "model.layers.87.mlp.experts.62.up_proj", "model.layers.87.mlp.experts.63.up_proj", "model.layers.87.mlp.experts.64.up_proj", "model.layers.87.mlp.experts.65.up_proj", "model.layers.87.mlp.experts.66.up_proj", "model.layers.87.mlp.experts.67.up_proj", "model.layers.87.mlp.experts.68.up_proj", "model.layers.87.mlp.experts.69.up_proj", "model.layers.87.mlp.experts.70.up_proj", "model.layers.87.mlp.experts.71.up_proj", "model.layers.87.mlp.experts.72.up_proj", "model.layers.87.mlp.experts.73.up_proj", "model.layers.87.mlp.experts.74.up_proj", "model.layers.87.mlp.experts.75.up_proj", "model.layers.87.mlp.experts.76.up_proj", "model.layers.87.mlp.experts.77.up_proj", "model.layers.87.mlp.experts.78.up_proj", "model.layers.87.mlp.experts.79.up_proj", "model.layers.87.mlp.experts.80.up_proj", "model.layers.87.mlp.experts.81.up_proj", "model.layers.87.mlp.experts.82.up_proj", "model.layers.87.mlp.experts.83.up_proj", "model.layers.87.mlp.experts.84.up_proj", "model.layers.87.mlp.experts.85.up_proj", "model.layers.87.mlp.experts.86.up_proj", "model.layers.87.mlp.experts.87.up_proj", "model.layers.87.mlp.experts.88.up_proj", "model.layers.87.mlp.experts.89.up_proj", "model.layers.87.mlp.experts.90.up_proj", "model.layers.87.mlp.experts.91.up_proj", "model.layers.87.mlp.experts.92.up_proj", "model.layers.87.mlp.experts.93.up_proj", "model.layers.87.mlp.experts.94.up_proj", "model.layers.87.mlp.experts.95.up_proj", "model.layers.87.mlp.experts.96.up_proj", "model.layers.87.mlp.experts.97.up_proj", "model.layers.87.mlp.experts.98.up_proj", "model.layers.87.mlp.experts.99.up_proj", "model.layers.87.mlp.experts.100.up_proj", "model.layers.87.mlp.experts.101.up_proj", "model.layers.87.mlp.experts.102.up_proj", "model.layers.87.mlp.experts.103.up_proj", "model.layers.87.mlp.experts.104.up_proj", "model.layers.87.mlp.experts.105.up_proj", "model.layers.87.mlp.experts.106.up_proj", "model.layers.87.mlp.experts.107.up_proj", "model.layers.87.mlp.experts.108.up_proj", "model.layers.87.mlp.experts.109.up_proj", "model.layers.87.mlp.experts.110.up_proj", "model.layers.87.mlp.experts.111.up_proj", "model.layers.87.mlp.experts.112.up_proj", "model.layers.87.mlp.experts.113.up_proj", "model.layers.87.mlp.experts.114.up_proj", "model.layers.87.mlp.experts.115.up_proj", "model.layers.87.mlp.experts.116.up_proj", "model.layers.87.mlp.experts.117.up_proj", "model.layers.87.mlp.experts.118.up_proj", "model.layers.87.mlp.experts.119.up_proj", "model.layers.87.mlp.experts.120.up_proj", "model.layers.87.mlp.experts.121.up_proj", "model.layers.87.mlp.experts.122.up_proj", "model.layers.87.mlp.experts.123.up_proj", "model.layers.87.mlp.experts.124.up_proj", "model.layers.87.mlp.experts.125.up_proj", "model.layers.87.mlp.experts.126.up_proj", "model.layers.87.mlp.experts.127.up_proj", "model.layers.87.mlp.experts.128.up_proj", "model.layers.87.mlp.experts.129.up_proj", "model.layers.87.mlp.experts.130.up_proj", "model.layers.87.mlp.experts.131.up_proj", "model.layers.87.mlp.experts.132.up_proj", "model.layers.87.mlp.experts.133.up_proj", "model.layers.87.mlp.experts.134.up_proj", "model.layers.87.mlp.experts.135.up_proj", "model.layers.87.mlp.experts.136.up_proj", "model.layers.87.mlp.experts.137.up_proj", "model.layers.87.mlp.experts.138.up_proj", "model.layers.87.mlp.experts.139.up_proj", "model.layers.87.mlp.experts.140.up_proj", "model.layers.87.mlp.experts.141.up_proj", "model.layers.87.mlp.experts.142.up_proj", "model.layers.87.mlp.experts.143.up_proj", "model.layers.87.mlp.experts.144.up_proj", "model.layers.87.mlp.experts.145.up_proj", "model.layers.87.mlp.experts.146.up_proj", "model.layers.87.mlp.experts.147.up_proj", "model.layers.87.mlp.experts.148.up_proj", "model.layers.87.mlp.experts.149.up_proj", "model.layers.87.mlp.experts.150.up_proj", "model.layers.87.mlp.experts.151.up_proj", "model.layers.87.mlp.experts.152.up_proj", "model.layers.87.mlp.experts.153.up_proj", "model.layers.87.mlp.experts.154.up_proj", "model.layers.87.mlp.experts.155.up_proj", "model.layers.87.mlp.experts.156.up_proj", "model.layers.87.mlp.experts.157.up_proj", "model.layers.87.mlp.experts.158.up_proj", "model.layers.87.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.0089617464690967e-06, "dbits": 5033164800 } ] }, { "idx": 524, "layers": [ "model.layers.87.mlp.experts.0.down_proj", "model.layers.87.mlp.experts.1.down_proj", "model.layers.87.mlp.experts.2.down_proj", "model.layers.87.mlp.experts.3.down_proj", "model.layers.87.mlp.experts.4.down_proj", "model.layers.87.mlp.experts.5.down_proj", "model.layers.87.mlp.experts.6.down_proj", "model.layers.87.mlp.experts.7.down_proj", "model.layers.87.mlp.experts.8.down_proj", "model.layers.87.mlp.experts.9.down_proj", "model.layers.87.mlp.experts.10.down_proj", "model.layers.87.mlp.experts.11.down_proj", "model.layers.87.mlp.experts.12.down_proj", "model.layers.87.mlp.experts.13.down_proj", "model.layers.87.mlp.experts.14.down_proj", "model.layers.87.mlp.experts.15.down_proj", "model.layers.87.mlp.experts.16.down_proj", "model.layers.87.mlp.experts.17.down_proj", "model.layers.87.mlp.experts.18.down_proj", "model.layers.87.mlp.experts.19.down_proj", "model.layers.87.mlp.experts.20.down_proj", "model.layers.87.mlp.experts.21.down_proj", "model.layers.87.mlp.experts.22.down_proj", "model.layers.87.mlp.experts.23.down_proj", "model.layers.87.mlp.experts.24.down_proj", "model.layers.87.mlp.experts.25.down_proj", "model.layers.87.mlp.experts.26.down_proj", "model.layers.87.mlp.experts.27.down_proj", "model.layers.87.mlp.experts.28.down_proj", "model.layers.87.mlp.experts.29.down_proj", "model.layers.87.mlp.experts.30.down_proj", "model.layers.87.mlp.experts.31.down_proj", "model.layers.87.mlp.experts.32.down_proj", "model.layers.87.mlp.experts.33.down_proj", "model.layers.87.mlp.experts.34.down_proj", "model.layers.87.mlp.experts.35.down_proj", "model.layers.87.mlp.experts.36.down_proj", "model.layers.87.mlp.experts.37.down_proj", "model.layers.87.mlp.experts.38.down_proj", "model.layers.87.mlp.experts.39.down_proj", "model.layers.87.mlp.experts.40.down_proj", "model.layers.87.mlp.experts.41.down_proj", "model.layers.87.mlp.experts.42.down_proj", "model.layers.87.mlp.experts.43.down_proj", "model.layers.87.mlp.experts.44.down_proj", "model.layers.87.mlp.experts.45.down_proj", "model.layers.87.mlp.experts.46.down_proj", "model.layers.87.mlp.experts.47.down_proj", "model.layers.87.mlp.experts.48.down_proj", "model.layers.87.mlp.experts.49.down_proj", "model.layers.87.mlp.experts.50.down_proj", "model.layers.87.mlp.experts.51.down_proj", "model.layers.87.mlp.experts.52.down_proj", "model.layers.87.mlp.experts.53.down_proj", "model.layers.87.mlp.experts.54.down_proj", "model.layers.87.mlp.experts.55.down_proj", "model.layers.87.mlp.experts.56.down_proj", "model.layers.87.mlp.experts.57.down_proj", "model.layers.87.mlp.experts.58.down_proj", "model.layers.87.mlp.experts.59.down_proj", "model.layers.87.mlp.experts.60.down_proj", "model.layers.87.mlp.experts.61.down_proj", "model.layers.87.mlp.experts.62.down_proj", "model.layers.87.mlp.experts.63.down_proj", "model.layers.87.mlp.experts.64.down_proj", "model.layers.87.mlp.experts.65.down_proj", "model.layers.87.mlp.experts.66.down_proj", "model.layers.87.mlp.experts.67.down_proj", "model.layers.87.mlp.experts.68.down_proj", "model.layers.87.mlp.experts.69.down_proj", "model.layers.87.mlp.experts.70.down_proj", "model.layers.87.mlp.experts.71.down_proj", "model.layers.87.mlp.experts.72.down_proj", "model.layers.87.mlp.experts.73.down_proj", "model.layers.87.mlp.experts.74.down_proj", "model.layers.87.mlp.experts.75.down_proj", "model.layers.87.mlp.experts.76.down_proj", "model.layers.87.mlp.experts.77.down_proj", "model.layers.87.mlp.experts.78.down_proj", "model.layers.87.mlp.experts.79.down_proj", "model.layers.87.mlp.experts.80.down_proj", "model.layers.87.mlp.experts.81.down_proj", "model.layers.87.mlp.experts.82.down_proj", "model.layers.87.mlp.experts.83.down_proj", "model.layers.87.mlp.experts.84.down_proj", "model.layers.87.mlp.experts.85.down_proj", "model.layers.87.mlp.experts.86.down_proj", "model.layers.87.mlp.experts.87.down_proj", "model.layers.87.mlp.experts.88.down_proj", "model.layers.87.mlp.experts.89.down_proj", "model.layers.87.mlp.experts.90.down_proj", "model.layers.87.mlp.experts.91.down_proj", "model.layers.87.mlp.experts.92.down_proj", "model.layers.87.mlp.experts.93.down_proj", "model.layers.87.mlp.experts.94.down_proj", "model.layers.87.mlp.experts.95.down_proj", "model.layers.87.mlp.experts.96.down_proj", "model.layers.87.mlp.experts.97.down_proj", "model.layers.87.mlp.experts.98.down_proj", "model.layers.87.mlp.experts.99.down_proj", "model.layers.87.mlp.experts.100.down_proj", "model.layers.87.mlp.experts.101.down_proj", "model.layers.87.mlp.experts.102.down_proj", "model.layers.87.mlp.experts.103.down_proj", "model.layers.87.mlp.experts.104.down_proj", "model.layers.87.mlp.experts.105.down_proj", "model.layers.87.mlp.experts.106.down_proj", "model.layers.87.mlp.experts.107.down_proj", "model.layers.87.mlp.experts.108.down_proj", "model.layers.87.mlp.experts.109.down_proj", "model.layers.87.mlp.experts.110.down_proj", "model.layers.87.mlp.experts.111.down_proj", "model.layers.87.mlp.experts.112.down_proj", "model.layers.87.mlp.experts.113.down_proj", "model.layers.87.mlp.experts.114.down_proj", "model.layers.87.mlp.experts.115.down_proj", "model.layers.87.mlp.experts.116.down_proj", "model.layers.87.mlp.experts.117.down_proj", "model.layers.87.mlp.experts.118.down_proj", "model.layers.87.mlp.experts.119.down_proj", "model.layers.87.mlp.experts.120.down_proj", "model.layers.87.mlp.experts.121.down_proj", "model.layers.87.mlp.experts.122.down_proj", "model.layers.87.mlp.experts.123.down_proj", "model.layers.87.mlp.experts.124.down_proj", "model.layers.87.mlp.experts.125.down_proj", "model.layers.87.mlp.experts.126.down_proj", "model.layers.87.mlp.experts.127.down_proj", "model.layers.87.mlp.experts.128.down_proj", "model.layers.87.mlp.experts.129.down_proj", "model.layers.87.mlp.experts.130.down_proj", "model.layers.87.mlp.experts.131.down_proj", "model.layers.87.mlp.experts.132.down_proj", "model.layers.87.mlp.experts.133.down_proj", "model.layers.87.mlp.experts.134.down_proj", "model.layers.87.mlp.experts.135.down_proj", "model.layers.87.mlp.experts.136.down_proj", "model.layers.87.mlp.experts.137.down_proj", "model.layers.87.mlp.experts.138.down_proj", "model.layers.87.mlp.experts.139.down_proj", "model.layers.87.mlp.experts.140.down_proj", "model.layers.87.mlp.experts.141.down_proj", "model.layers.87.mlp.experts.142.down_proj", "model.layers.87.mlp.experts.143.down_proj", "model.layers.87.mlp.experts.144.down_proj", "model.layers.87.mlp.experts.145.down_proj", "model.layers.87.mlp.experts.146.down_proj", "model.layers.87.mlp.experts.147.down_proj", "model.layers.87.mlp.experts.148.down_proj", "model.layers.87.mlp.experts.149.down_proj", "model.layers.87.mlp.experts.150.down_proj", "model.layers.87.mlp.experts.151.down_proj", "model.layers.87.mlp.experts.152.down_proj", "model.layers.87.mlp.experts.153.down_proj", "model.layers.87.mlp.experts.154.down_proj", "model.layers.87.mlp.experts.155.down_proj", "model.layers.87.mlp.experts.156.down_proj", "model.layers.87.mlp.experts.157.down_proj", "model.layers.87.mlp.experts.158.down_proj", "model.layers.87.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.4491117326543169e-06, "dbits": 2516582400 } ] }, { "idx": 525, "layers": [ "model.layers.88.self_attn.q_proj" ], "candidates": [ { "dkld": 1.0587362339719696e-06, "dbits": 125829120 } ] }, { "idx": 526, "layers": [ "model.layers.88.self_attn.k_proj", "model.layers.88.self_attn.v_proj" ], "candidates": [ { "dkld": -5.935682565905154e-06, "dbits": 20971520 } ] }, { "idx": 527, "layers": [ "model.layers.88.self_attn.o_proj" ], "candidates": [ { "dkld": -5.908572347834489e-07, "dbits": 125829120 } ] }, { "idx": 528, "layers": [ "model.layers.88.mlp.shared_experts.gate_proj", "model.layers.88.mlp.shared_experts.up_proj", "model.layers.88.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.5450583305210092e-06, "dbits": 47185920 } ] }, { "idx": 529, "layers": [ "model.layers.88.mlp.experts.0.gate_proj", "model.layers.88.mlp.experts.1.gate_proj", "model.layers.88.mlp.experts.2.gate_proj", "model.layers.88.mlp.experts.3.gate_proj", "model.layers.88.mlp.experts.4.gate_proj", "model.layers.88.mlp.experts.5.gate_proj", "model.layers.88.mlp.experts.6.gate_proj", "model.layers.88.mlp.experts.7.gate_proj", "model.layers.88.mlp.experts.8.gate_proj", "model.layers.88.mlp.experts.9.gate_proj", "model.layers.88.mlp.experts.10.gate_proj", "model.layers.88.mlp.experts.11.gate_proj", "model.layers.88.mlp.experts.12.gate_proj", "model.layers.88.mlp.experts.13.gate_proj", "model.layers.88.mlp.experts.14.gate_proj", "model.layers.88.mlp.experts.15.gate_proj", "model.layers.88.mlp.experts.16.gate_proj", "model.layers.88.mlp.experts.17.gate_proj", "model.layers.88.mlp.experts.18.gate_proj", "model.layers.88.mlp.experts.19.gate_proj", "model.layers.88.mlp.experts.20.gate_proj", "model.layers.88.mlp.experts.21.gate_proj", "model.layers.88.mlp.experts.22.gate_proj", "model.layers.88.mlp.experts.23.gate_proj", "model.layers.88.mlp.experts.24.gate_proj", "model.layers.88.mlp.experts.25.gate_proj", "model.layers.88.mlp.experts.26.gate_proj", "model.layers.88.mlp.experts.27.gate_proj", "model.layers.88.mlp.experts.28.gate_proj", "model.layers.88.mlp.experts.29.gate_proj", "model.layers.88.mlp.experts.30.gate_proj", "model.layers.88.mlp.experts.31.gate_proj", "model.layers.88.mlp.experts.32.gate_proj", "model.layers.88.mlp.experts.33.gate_proj", "model.layers.88.mlp.experts.34.gate_proj", "model.layers.88.mlp.experts.35.gate_proj", "model.layers.88.mlp.experts.36.gate_proj", "model.layers.88.mlp.experts.37.gate_proj", "model.layers.88.mlp.experts.38.gate_proj", "model.layers.88.mlp.experts.39.gate_proj", "model.layers.88.mlp.experts.40.gate_proj", "model.layers.88.mlp.experts.41.gate_proj", "model.layers.88.mlp.experts.42.gate_proj", "model.layers.88.mlp.experts.43.gate_proj", "model.layers.88.mlp.experts.44.gate_proj", "model.layers.88.mlp.experts.45.gate_proj", "model.layers.88.mlp.experts.46.gate_proj", "model.layers.88.mlp.experts.47.gate_proj", "model.layers.88.mlp.experts.48.gate_proj", "model.layers.88.mlp.experts.49.gate_proj", "model.layers.88.mlp.experts.50.gate_proj", "model.layers.88.mlp.experts.51.gate_proj", "model.layers.88.mlp.experts.52.gate_proj", "model.layers.88.mlp.experts.53.gate_proj", "model.layers.88.mlp.experts.54.gate_proj", "model.layers.88.mlp.experts.55.gate_proj", "model.layers.88.mlp.experts.56.gate_proj", "model.layers.88.mlp.experts.57.gate_proj", "model.layers.88.mlp.experts.58.gate_proj", "model.layers.88.mlp.experts.59.gate_proj", "model.layers.88.mlp.experts.60.gate_proj", "model.layers.88.mlp.experts.61.gate_proj", "model.layers.88.mlp.experts.62.gate_proj", "model.layers.88.mlp.experts.63.gate_proj", "model.layers.88.mlp.experts.64.gate_proj", "model.layers.88.mlp.experts.65.gate_proj", "model.layers.88.mlp.experts.66.gate_proj", "model.layers.88.mlp.experts.67.gate_proj", "model.layers.88.mlp.experts.68.gate_proj", "model.layers.88.mlp.experts.69.gate_proj", "model.layers.88.mlp.experts.70.gate_proj", "model.layers.88.mlp.experts.71.gate_proj", "model.layers.88.mlp.experts.72.gate_proj", "model.layers.88.mlp.experts.73.gate_proj", "model.layers.88.mlp.experts.74.gate_proj", "model.layers.88.mlp.experts.75.gate_proj", "model.layers.88.mlp.experts.76.gate_proj", "model.layers.88.mlp.experts.77.gate_proj", "model.layers.88.mlp.experts.78.gate_proj", "model.layers.88.mlp.experts.79.gate_proj", "model.layers.88.mlp.experts.80.gate_proj", "model.layers.88.mlp.experts.81.gate_proj", "model.layers.88.mlp.experts.82.gate_proj", "model.layers.88.mlp.experts.83.gate_proj", "model.layers.88.mlp.experts.84.gate_proj", "model.layers.88.mlp.experts.85.gate_proj", "model.layers.88.mlp.experts.86.gate_proj", "model.layers.88.mlp.experts.87.gate_proj", "model.layers.88.mlp.experts.88.gate_proj", "model.layers.88.mlp.experts.89.gate_proj", "model.layers.88.mlp.experts.90.gate_proj", "model.layers.88.mlp.experts.91.gate_proj", "model.layers.88.mlp.experts.92.gate_proj", "model.layers.88.mlp.experts.93.gate_proj", "model.layers.88.mlp.experts.94.gate_proj", "model.layers.88.mlp.experts.95.gate_proj", "model.layers.88.mlp.experts.96.gate_proj", "model.layers.88.mlp.experts.97.gate_proj", "model.layers.88.mlp.experts.98.gate_proj", "model.layers.88.mlp.experts.99.gate_proj", "model.layers.88.mlp.experts.100.gate_proj", "model.layers.88.mlp.experts.101.gate_proj", "model.layers.88.mlp.experts.102.gate_proj", "model.layers.88.mlp.experts.103.gate_proj", "model.layers.88.mlp.experts.104.gate_proj", "model.layers.88.mlp.experts.105.gate_proj", "model.layers.88.mlp.experts.106.gate_proj", "model.layers.88.mlp.experts.107.gate_proj", "model.layers.88.mlp.experts.108.gate_proj", "model.layers.88.mlp.experts.109.gate_proj", "model.layers.88.mlp.experts.110.gate_proj", "model.layers.88.mlp.experts.111.gate_proj", "model.layers.88.mlp.experts.112.gate_proj", "model.layers.88.mlp.experts.113.gate_proj", "model.layers.88.mlp.experts.114.gate_proj", "model.layers.88.mlp.experts.115.gate_proj", "model.layers.88.mlp.experts.116.gate_proj", "model.layers.88.mlp.experts.117.gate_proj", "model.layers.88.mlp.experts.118.gate_proj", "model.layers.88.mlp.experts.119.gate_proj", "model.layers.88.mlp.experts.120.gate_proj", "model.layers.88.mlp.experts.121.gate_proj", "model.layers.88.mlp.experts.122.gate_proj", "model.layers.88.mlp.experts.123.gate_proj", "model.layers.88.mlp.experts.124.gate_proj", "model.layers.88.mlp.experts.125.gate_proj", "model.layers.88.mlp.experts.126.gate_proj", "model.layers.88.mlp.experts.127.gate_proj", "model.layers.88.mlp.experts.128.gate_proj", "model.layers.88.mlp.experts.129.gate_proj", "model.layers.88.mlp.experts.130.gate_proj", "model.layers.88.mlp.experts.131.gate_proj", "model.layers.88.mlp.experts.132.gate_proj", "model.layers.88.mlp.experts.133.gate_proj", "model.layers.88.mlp.experts.134.gate_proj", "model.layers.88.mlp.experts.135.gate_proj", "model.layers.88.mlp.experts.136.gate_proj", "model.layers.88.mlp.experts.137.gate_proj", "model.layers.88.mlp.experts.138.gate_proj", "model.layers.88.mlp.experts.139.gate_proj", "model.layers.88.mlp.experts.140.gate_proj", "model.layers.88.mlp.experts.141.gate_proj", "model.layers.88.mlp.experts.142.gate_proj", "model.layers.88.mlp.experts.143.gate_proj", "model.layers.88.mlp.experts.144.gate_proj", "model.layers.88.mlp.experts.145.gate_proj", "model.layers.88.mlp.experts.146.gate_proj", "model.layers.88.mlp.experts.147.gate_proj", "model.layers.88.mlp.experts.148.gate_proj", "model.layers.88.mlp.experts.149.gate_proj", "model.layers.88.mlp.experts.150.gate_proj", "model.layers.88.mlp.experts.151.gate_proj", "model.layers.88.mlp.experts.152.gate_proj", "model.layers.88.mlp.experts.153.gate_proj", "model.layers.88.mlp.experts.154.gate_proj", "model.layers.88.mlp.experts.155.gate_proj", "model.layers.88.mlp.experts.156.gate_proj", "model.layers.88.mlp.experts.157.gate_proj", "model.layers.88.mlp.experts.158.gate_proj", "model.layers.88.mlp.experts.159.gate_proj", "model.layers.88.mlp.experts.0.up_proj", "model.layers.88.mlp.experts.1.up_proj", "model.layers.88.mlp.experts.2.up_proj", "model.layers.88.mlp.experts.3.up_proj", "model.layers.88.mlp.experts.4.up_proj", "model.layers.88.mlp.experts.5.up_proj", "model.layers.88.mlp.experts.6.up_proj", "model.layers.88.mlp.experts.7.up_proj", "model.layers.88.mlp.experts.8.up_proj", "model.layers.88.mlp.experts.9.up_proj", "model.layers.88.mlp.experts.10.up_proj", "model.layers.88.mlp.experts.11.up_proj", "model.layers.88.mlp.experts.12.up_proj", "model.layers.88.mlp.experts.13.up_proj", "model.layers.88.mlp.experts.14.up_proj", "model.layers.88.mlp.experts.15.up_proj", "model.layers.88.mlp.experts.16.up_proj", "model.layers.88.mlp.experts.17.up_proj", "model.layers.88.mlp.experts.18.up_proj", "model.layers.88.mlp.experts.19.up_proj", "model.layers.88.mlp.experts.20.up_proj", "model.layers.88.mlp.experts.21.up_proj", "model.layers.88.mlp.experts.22.up_proj", "model.layers.88.mlp.experts.23.up_proj", "model.layers.88.mlp.experts.24.up_proj", "model.layers.88.mlp.experts.25.up_proj", "model.layers.88.mlp.experts.26.up_proj", "model.layers.88.mlp.experts.27.up_proj", "model.layers.88.mlp.experts.28.up_proj", "model.layers.88.mlp.experts.29.up_proj", "model.layers.88.mlp.experts.30.up_proj", "model.layers.88.mlp.experts.31.up_proj", "model.layers.88.mlp.experts.32.up_proj", "model.layers.88.mlp.experts.33.up_proj", "model.layers.88.mlp.experts.34.up_proj", "model.layers.88.mlp.experts.35.up_proj", "model.layers.88.mlp.experts.36.up_proj", "model.layers.88.mlp.experts.37.up_proj", "model.layers.88.mlp.experts.38.up_proj", "model.layers.88.mlp.experts.39.up_proj", "model.layers.88.mlp.experts.40.up_proj", "model.layers.88.mlp.experts.41.up_proj", "model.layers.88.mlp.experts.42.up_proj", "model.layers.88.mlp.experts.43.up_proj", "model.layers.88.mlp.experts.44.up_proj", "model.layers.88.mlp.experts.45.up_proj", "model.layers.88.mlp.experts.46.up_proj", "model.layers.88.mlp.experts.47.up_proj", "model.layers.88.mlp.experts.48.up_proj", "model.layers.88.mlp.experts.49.up_proj", "model.layers.88.mlp.experts.50.up_proj", "model.layers.88.mlp.experts.51.up_proj", "model.layers.88.mlp.experts.52.up_proj", "model.layers.88.mlp.experts.53.up_proj", "model.layers.88.mlp.experts.54.up_proj", "model.layers.88.mlp.experts.55.up_proj", "model.layers.88.mlp.experts.56.up_proj", "model.layers.88.mlp.experts.57.up_proj", "model.layers.88.mlp.experts.58.up_proj", "model.layers.88.mlp.experts.59.up_proj", "model.layers.88.mlp.experts.60.up_proj", "model.layers.88.mlp.experts.61.up_proj", "model.layers.88.mlp.experts.62.up_proj", "model.layers.88.mlp.experts.63.up_proj", "model.layers.88.mlp.experts.64.up_proj", "model.layers.88.mlp.experts.65.up_proj", "model.layers.88.mlp.experts.66.up_proj", "model.layers.88.mlp.experts.67.up_proj", "model.layers.88.mlp.experts.68.up_proj", "model.layers.88.mlp.experts.69.up_proj", "model.layers.88.mlp.experts.70.up_proj", "model.layers.88.mlp.experts.71.up_proj", "model.layers.88.mlp.experts.72.up_proj", "model.layers.88.mlp.experts.73.up_proj", "model.layers.88.mlp.experts.74.up_proj", "model.layers.88.mlp.experts.75.up_proj", "model.layers.88.mlp.experts.76.up_proj", "model.layers.88.mlp.experts.77.up_proj", "model.layers.88.mlp.experts.78.up_proj", "model.layers.88.mlp.experts.79.up_proj", "model.layers.88.mlp.experts.80.up_proj", "model.layers.88.mlp.experts.81.up_proj", "model.layers.88.mlp.experts.82.up_proj", "model.layers.88.mlp.experts.83.up_proj", "model.layers.88.mlp.experts.84.up_proj", "model.layers.88.mlp.experts.85.up_proj", "model.layers.88.mlp.experts.86.up_proj", "model.layers.88.mlp.experts.87.up_proj", "model.layers.88.mlp.experts.88.up_proj", "model.layers.88.mlp.experts.89.up_proj", "model.layers.88.mlp.experts.90.up_proj", "model.layers.88.mlp.experts.91.up_proj", "model.layers.88.mlp.experts.92.up_proj", "model.layers.88.mlp.experts.93.up_proj", "model.layers.88.mlp.experts.94.up_proj", "model.layers.88.mlp.experts.95.up_proj", "model.layers.88.mlp.experts.96.up_proj", "model.layers.88.mlp.experts.97.up_proj", "model.layers.88.mlp.experts.98.up_proj", "model.layers.88.mlp.experts.99.up_proj", "model.layers.88.mlp.experts.100.up_proj", "model.layers.88.mlp.experts.101.up_proj", "model.layers.88.mlp.experts.102.up_proj", "model.layers.88.mlp.experts.103.up_proj", "model.layers.88.mlp.experts.104.up_proj", "model.layers.88.mlp.experts.105.up_proj", "model.layers.88.mlp.experts.106.up_proj", "model.layers.88.mlp.experts.107.up_proj", "model.layers.88.mlp.experts.108.up_proj", "model.layers.88.mlp.experts.109.up_proj", "model.layers.88.mlp.experts.110.up_proj", "model.layers.88.mlp.experts.111.up_proj", "model.layers.88.mlp.experts.112.up_proj", "model.layers.88.mlp.experts.113.up_proj", "model.layers.88.mlp.experts.114.up_proj", "model.layers.88.mlp.experts.115.up_proj", "model.layers.88.mlp.experts.116.up_proj", "model.layers.88.mlp.experts.117.up_proj", "model.layers.88.mlp.experts.118.up_proj", "model.layers.88.mlp.experts.119.up_proj", "model.layers.88.mlp.experts.120.up_proj", "model.layers.88.mlp.experts.121.up_proj", "model.layers.88.mlp.experts.122.up_proj", "model.layers.88.mlp.experts.123.up_proj", "model.layers.88.mlp.experts.124.up_proj", "model.layers.88.mlp.experts.125.up_proj", "model.layers.88.mlp.experts.126.up_proj", "model.layers.88.mlp.experts.127.up_proj", "model.layers.88.mlp.experts.128.up_proj", "model.layers.88.mlp.experts.129.up_proj", "model.layers.88.mlp.experts.130.up_proj", "model.layers.88.mlp.experts.131.up_proj", "model.layers.88.mlp.experts.132.up_proj", "model.layers.88.mlp.experts.133.up_proj", "model.layers.88.mlp.experts.134.up_proj", "model.layers.88.mlp.experts.135.up_proj", "model.layers.88.mlp.experts.136.up_proj", "model.layers.88.mlp.experts.137.up_proj", "model.layers.88.mlp.experts.138.up_proj", "model.layers.88.mlp.experts.139.up_proj", "model.layers.88.mlp.experts.140.up_proj", "model.layers.88.mlp.experts.141.up_proj", "model.layers.88.mlp.experts.142.up_proj", "model.layers.88.mlp.experts.143.up_proj", "model.layers.88.mlp.experts.144.up_proj", "model.layers.88.mlp.experts.145.up_proj", "model.layers.88.mlp.experts.146.up_proj", "model.layers.88.mlp.experts.147.up_proj", "model.layers.88.mlp.experts.148.up_proj", "model.layers.88.mlp.experts.149.up_proj", "model.layers.88.mlp.experts.150.up_proj", "model.layers.88.mlp.experts.151.up_proj", "model.layers.88.mlp.experts.152.up_proj", "model.layers.88.mlp.experts.153.up_proj", "model.layers.88.mlp.experts.154.up_proj", "model.layers.88.mlp.experts.155.up_proj", "model.layers.88.mlp.experts.156.up_proj", "model.layers.88.mlp.experts.157.up_proj", "model.layers.88.mlp.experts.158.up_proj", "model.layers.88.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 2.4861306883394935e-06, "dbits": 5033164800 } ] }, { "idx": 530, "layers": [ "model.layers.88.mlp.experts.0.down_proj", "model.layers.88.mlp.experts.1.down_proj", "model.layers.88.mlp.experts.2.down_proj", "model.layers.88.mlp.experts.3.down_proj", "model.layers.88.mlp.experts.4.down_proj", "model.layers.88.mlp.experts.5.down_proj", "model.layers.88.mlp.experts.6.down_proj", "model.layers.88.mlp.experts.7.down_proj", "model.layers.88.mlp.experts.8.down_proj", "model.layers.88.mlp.experts.9.down_proj", "model.layers.88.mlp.experts.10.down_proj", "model.layers.88.mlp.experts.11.down_proj", "model.layers.88.mlp.experts.12.down_proj", "model.layers.88.mlp.experts.13.down_proj", "model.layers.88.mlp.experts.14.down_proj", "model.layers.88.mlp.experts.15.down_proj", "model.layers.88.mlp.experts.16.down_proj", "model.layers.88.mlp.experts.17.down_proj", "model.layers.88.mlp.experts.18.down_proj", "model.layers.88.mlp.experts.19.down_proj", "model.layers.88.mlp.experts.20.down_proj", "model.layers.88.mlp.experts.21.down_proj", "model.layers.88.mlp.experts.22.down_proj", "model.layers.88.mlp.experts.23.down_proj", "model.layers.88.mlp.experts.24.down_proj", "model.layers.88.mlp.experts.25.down_proj", "model.layers.88.mlp.experts.26.down_proj", "model.layers.88.mlp.experts.27.down_proj", "model.layers.88.mlp.experts.28.down_proj", "model.layers.88.mlp.experts.29.down_proj", "model.layers.88.mlp.experts.30.down_proj", "model.layers.88.mlp.experts.31.down_proj", "model.layers.88.mlp.experts.32.down_proj", "model.layers.88.mlp.experts.33.down_proj", "model.layers.88.mlp.experts.34.down_proj", "model.layers.88.mlp.experts.35.down_proj", "model.layers.88.mlp.experts.36.down_proj", "model.layers.88.mlp.experts.37.down_proj", "model.layers.88.mlp.experts.38.down_proj", "model.layers.88.mlp.experts.39.down_proj", "model.layers.88.mlp.experts.40.down_proj", "model.layers.88.mlp.experts.41.down_proj", "model.layers.88.mlp.experts.42.down_proj", "model.layers.88.mlp.experts.43.down_proj", "model.layers.88.mlp.experts.44.down_proj", "model.layers.88.mlp.experts.45.down_proj", "model.layers.88.mlp.experts.46.down_proj", "model.layers.88.mlp.experts.47.down_proj", "model.layers.88.mlp.experts.48.down_proj", "model.layers.88.mlp.experts.49.down_proj", "model.layers.88.mlp.experts.50.down_proj", "model.layers.88.mlp.experts.51.down_proj", "model.layers.88.mlp.experts.52.down_proj", "model.layers.88.mlp.experts.53.down_proj", "model.layers.88.mlp.experts.54.down_proj", "model.layers.88.mlp.experts.55.down_proj", "model.layers.88.mlp.experts.56.down_proj", "model.layers.88.mlp.experts.57.down_proj", "model.layers.88.mlp.experts.58.down_proj", "model.layers.88.mlp.experts.59.down_proj", "model.layers.88.mlp.experts.60.down_proj", "model.layers.88.mlp.experts.61.down_proj", "model.layers.88.mlp.experts.62.down_proj", "model.layers.88.mlp.experts.63.down_proj", "model.layers.88.mlp.experts.64.down_proj", "model.layers.88.mlp.experts.65.down_proj", "model.layers.88.mlp.experts.66.down_proj", "model.layers.88.mlp.experts.67.down_proj", "model.layers.88.mlp.experts.68.down_proj", "model.layers.88.mlp.experts.69.down_proj", "model.layers.88.mlp.experts.70.down_proj", "model.layers.88.mlp.experts.71.down_proj", "model.layers.88.mlp.experts.72.down_proj", "model.layers.88.mlp.experts.73.down_proj", "model.layers.88.mlp.experts.74.down_proj", "model.layers.88.mlp.experts.75.down_proj", "model.layers.88.mlp.experts.76.down_proj", "model.layers.88.mlp.experts.77.down_proj", "model.layers.88.mlp.experts.78.down_proj", "model.layers.88.mlp.experts.79.down_proj", "model.layers.88.mlp.experts.80.down_proj", "model.layers.88.mlp.experts.81.down_proj", "model.layers.88.mlp.experts.82.down_proj", "model.layers.88.mlp.experts.83.down_proj", "model.layers.88.mlp.experts.84.down_proj", "model.layers.88.mlp.experts.85.down_proj", "model.layers.88.mlp.experts.86.down_proj", "model.layers.88.mlp.experts.87.down_proj", "model.layers.88.mlp.experts.88.down_proj", "model.layers.88.mlp.experts.89.down_proj", "model.layers.88.mlp.experts.90.down_proj", "model.layers.88.mlp.experts.91.down_proj", "model.layers.88.mlp.experts.92.down_proj", "model.layers.88.mlp.experts.93.down_proj", "model.layers.88.mlp.experts.94.down_proj", "model.layers.88.mlp.experts.95.down_proj", "model.layers.88.mlp.experts.96.down_proj", "model.layers.88.mlp.experts.97.down_proj", "model.layers.88.mlp.experts.98.down_proj", "model.layers.88.mlp.experts.99.down_proj", "model.layers.88.mlp.experts.100.down_proj", "model.layers.88.mlp.experts.101.down_proj", "model.layers.88.mlp.experts.102.down_proj", "model.layers.88.mlp.experts.103.down_proj", "model.layers.88.mlp.experts.104.down_proj", "model.layers.88.mlp.experts.105.down_proj", "model.layers.88.mlp.experts.106.down_proj", "model.layers.88.mlp.experts.107.down_proj", "model.layers.88.mlp.experts.108.down_proj", "model.layers.88.mlp.experts.109.down_proj", "model.layers.88.mlp.experts.110.down_proj", "model.layers.88.mlp.experts.111.down_proj", "model.layers.88.mlp.experts.112.down_proj", "model.layers.88.mlp.experts.113.down_proj", "model.layers.88.mlp.experts.114.down_proj", "model.layers.88.mlp.experts.115.down_proj", "model.layers.88.mlp.experts.116.down_proj", "model.layers.88.mlp.experts.117.down_proj", "model.layers.88.mlp.experts.118.down_proj", "model.layers.88.mlp.experts.119.down_proj", "model.layers.88.mlp.experts.120.down_proj", "model.layers.88.mlp.experts.121.down_proj", "model.layers.88.mlp.experts.122.down_proj", "model.layers.88.mlp.experts.123.down_proj", "model.layers.88.mlp.experts.124.down_proj", "model.layers.88.mlp.experts.125.down_proj", "model.layers.88.mlp.experts.126.down_proj", "model.layers.88.mlp.experts.127.down_proj", "model.layers.88.mlp.experts.128.down_proj", "model.layers.88.mlp.experts.129.down_proj", "model.layers.88.mlp.experts.130.down_proj", "model.layers.88.mlp.experts.131.down_proj", "model.layers.88.mlp.experts.132.down_proj", "model.layers.88.mlp.experts.133.down_proj", "model.layers.88.mlp.experts.134.down_proj", "model.layers.88.mlp.experts.135.down_proj", "model.layers.88.mlp.experts.136.down_proj", "model.layers.88.mlp.experts.137.down_proj", "model.layers.88.mlp.experts.138.down_proj", "model.layers.88.mlp.experts.139.down_proj", "model.layers.88.mlp.experts.140.down_proj", "model.layers.88.mlp.experts.141.down_proj", "model.layers.88.mlp.experts.142.down_proj", "model.layers.88.mlp.experts.143.down_proj", "model.layers.88.mlp.experts.144.down_proj", "model.layers.88.mlp.experts.145.down_proj", "model.layers.88.mlp.experts.146.down_proj", "model.layers.88.mlp.experts.147.down_proj", "model.layers.88.mlp.experts.148.down_proj", "model.layers.88.mlp.experts.149.down_proj", "model.layers.88.mlp.experts.150.down_proj", "model.layers.88.mlp.experts.151.down_proj", "model.layers.88.mlp.experts.152.down_proj", "model.layers.88.mlp.experts.153.down_proj", "model.layers.88.mlp.experts.154.down_proj", "model.layers.88.mlp.experts.155.down_proj", "model.layers.88.mlp.experts.156.down_proj", "model.layers.88.mlp.experts.157.down_proj", "model.layers.88.mlp.experts.158.down_proj", "model.layers.88.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 7.973692845552726e-07, "dbits": 2516582400 } ] }, { "idx": 531, "layers": [ "model.layers.89.self_attn.q_proj" ], "candidates": [ { "dkld": 6.918882718310194e-07, "dbits": 125829120 } ] }, { "idx": 532, "layers": [ "model.layers.89.self_attn.k_proj", "model.layers.89.self_attn.v_proj" ], "candidates": [ { "dkld": -1.9722621073015078e-05, "dbits": 20971520 } ] }, { "idx": 533, "layers": [ "model.layers.89.self_attn.o_proj" ], "candidates": [ { "dkld": -5.475711077451272e-07, "dbits": 125829120 } ] }, { "idx": 534, "layers": [ "model.layers.89.mlp.shared_experts.gate_proj", "model.layers.89.mlp.shared_experts.up_proj", "model.layers.89.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -1.652282662689469e-07, "dbits": 47185920 } ] }, { "idx": 535, "layers": [ "model.layers.89.mlp.experts.0.gate_proj", "model.layers.89.mlp.experts.1.gate_proj", "model.layers.89.mlp.experts.2.gate_proj", "model.layers.89.mlp.experts.3.gate_proj", "model.layers.89.mlp.experts.4.gate_proj", "model.layers.89.mlp.experts.5.gate_proj", "model.layers.89.mlp.experts.6.gate_proj", "model.layers.89.mlp.experts.7.gate_proj", "model.layers.89.mlp.experts.8.gate_proj", "model.layers.89.mlp.experts.9.gate_proj", "model.layers.89.mlp.experts.10.gate_proj", "model.layers.89.mlp.experts.11.gate_proj", "model.layers.89.mlp.experts.12.gate_proj", "model.layers.89.mlp.experts.13.gate_proj", "model.layers.89.mlp.experts.14.gate_proj", "model.layers.89.mlp.experts.15.gate_proj", "model.layers.89.mlp.experts.16.gate_proj", "model.layers.89.mlp.experts.17.gate_proj", "model.layers.89.mlp.experts.18.gate_proj", "model.layers.89.mlp.experts.19.gate_proj", "model.layers.89.mlp.experts.20.gate_proj", "model.layers.89.mlp.experts.21.gate_proj", "model.layers.89.mlp.experts.22.gate_proj", "model.layers.89.mlp.experts.23.gate_proj", "model.layers.89.mlp.experts.24.gate_proj", "model.layers.89.mlp.experts.25.gate_proj", "model.layers.89.mlp.experts.26.gate_proj", "model.layers.89.mlp.experts.27.gate_proj", "model.layers.89.mlp.experts.28.gate_proj", "model.layers.89.mlp.experts.29.gate_proj", "model.layers.89.mlp.experts.30.gate_proj", "model.layers.89.mlp.experts.31.gate_proj", "model.layers.89.mlp.experts.32.gate_proj", "model.layers.89.mlp.experts.33.gate_proj", "model.layers.89.mlp.experts.34.gate_proj", "model.layers.89.mlp.experts.35.gate_proj", "model.layers.89.mlp.experts.36.gate_proj", "model.layers.89.mlp.experts.37.gate_proj", "model.layers.89.mlp.experts.38.gate_proj", "model.layers.89.mlp.experts.39.gate_proj", "model.layers.89.mlp.experts.40.gate_proj", "model.layers.89.mlp.experts.41.gate_proj", "model.layers.89.mlp.experts.42.gate_proj", "model.layers.89.mlp.experts.43.gate_proj", "model.layers.89.mlp.experts.44.gate_proj", "model.layers.89.mlp.experts.45.gate_proj", "model.layers.89.mlp.experts.46.gate_proj", "model.layers.89.mlp.experts.47.gate_proj", "model.layers.89.mlp.experts.48.gate_proj", "model.layers.89.mlp.experts.49.gate_proj", "model.layers.89.mlp.experts.50.gate_proj", "model.layers.89.mlp.experts.51.gate_proj", "model.layers.89.mlp.experts.52.gate_proj", "model.layers.89.mlp.experts.53.gate_proj", "model.layers.89.mlp.experts.54.gate_proj", "model.layers.89.mlp.experts.55.gate_proj", "model.layers.89.mlp.experts.56.gate_proj", "model.layers.89.mlp.experts.57.gate_proj", "model.layers.89.mlp.experts.58.gate_proj", "model.layers.89.mlp.experts.59.gate_proj", "model.layers.89.mlp.experts.60.gate_proj", "model.layers.89.mlp.experts.61.gate_proj", "model.layers.89.mlp.experts.62.gate_proj", "model.layers.89.mlp.experts.63.gate_proj", "model.layers.89.mlp.experts.64.gate_proj", "model.layers.89.mlp.experts.65.gate_proj", "model.layers.89.mlp.experts.66.gate_proj", "model.layers.89.mlp.experts.67.gate_proj", "model.layers.89.mlp.experts.68.gate_proj", "model.layers.89.mlp.experts.69.gate_proj", "model.layers.89.mlp.experts.70.gate_proj", "model.layers.89.mlp.experts.71.gate_proj", "model.layers.89.mlp.experts.72.gate_proj", "model.layers.89.mlp.experts.73.gate_proj", "model.layers.89.mlp.experts.74.gate_proj", "model.layers.89.mlp.experts.75.gate_proj", "model.layers.89.mlp.experts.76.gate_proj", "model.layers.89.mlp.experts.77.gate_proj", "model.layers.89.mlp.experts.78.gate_proj", "model.layers.89.mlp.experts.79.gate_proj", "model.layers.89.mlp.experts.80.gate_proj", "model.layers.89.mlp.experts.81.gate_proj", "model.layers.89.mlp.experts.82.gate_proj", "model.layers.89.mlp.experts.83.gate_proj", "model.layers.89.mlp.experts.84.gate_proj", "model.layers.89.mlp.experts.85.gate_proj", "model.layers.89.mlp.experts.86.gate_proj", "model.layers.89.mlp.experts.87.gate_proj", "model.layers.89.mlp.experts.88.gate_proj", "model.layers.89.mlp.experts.89.gate_proj", "model.layers.89.mlp.experts.90.gate_proj", "model.layers.89.mlp.experts.91.gate_proj", "model.layers.89.mlp.experts.92.gate_proj", "model.layers.89.mlp.experts.93.gate_proj", "model.layers.89.mlp.experts.94.gate_proj", "model.layers.89.mlp.experts.95.gate_proj", "model.layers.89.mlp.experts.96.gate_proj", "model.layers.89.mlp.experts.97.gate_proj", "model.layers.89.mlp.experts.98.gate_proj", "model.layers.89.mlp.experts.99.gate_proj", "model.layers.89.mlp.experts.100.gate_proj", "model.layers.89.mlp.experts.101.gate_proj", "model.layers.89.mlp.experts.102.gate_proj", "model.layers.89.mlp.experts.103.gate_proj", "model.layers.89.mlp.experts.104.gate_proj", "model.layers.89.mlp.experts.105.gate_proj", "model.layers.89.mlp.experts.106.gate_proj", "model.layers.89.mlp.experts.107.gate_proj", "model.layers.89.mlp.experts.108.gate_proj", "model.layers.89.mlp.experts.109.gate_proj", "model.layers.89.mlp.experts.110.gate_proj", "model.layers.89.mlp.experts.111.gate_proj", "model.layers.89.mlp.experts.112.gate_proj", "model.layers.89.mlp.experts.113.gate_proj", "model.layers.89.mlp.experts.114.gate_proj", "model.layers.89.mlp.experts.115.gate_proj", "model.layers.89.mlp.experts.116.gate_proj", "model.layers.89.mlp.experts.117.gate_proj", "model.layers.89.mlp.experts.118.gate_proj", "model.layers.89.mlp.experts.119.gate_proj", "model.layers.89.mlp.experts.120.gate_proj", "model.layers.89.mlp.experts.121.gate_proj", "model.layers.89.mlp.experts.122.gate_proj", "model.layers.89.mlp.experts.123.gate_proj", "model.layers.89.mlp.experts.124.gate_proj", "model.layers.89.mlp.experts.125.gate_proj", "model.layers.89.mlp.experts.126.gate_proj", "model.layers.89.mlp.experts.127.gate_proj", "model.layers.89.mlp.experts.128.gate_proj", "model.layers.89.mlp.experts.129.gate_proj", "model.layers.89.mlp.experts.130.gate_proj", "model.layers.89.mlp.experts.131.gate_proj", "model.layers.89.mlp.experts.132.gate_proj", "model.layers.89.mlp.experts.133.gate_proj", "model.layers.89.mlp.experts.134.gate_proj", "model.layers.89.mlp.experts.135.gate_proj", "model.layers.89.mlp.experts.136.gate_proj", "model.layers.89.mlp.experts.137.gate_proj", "model.layers.89.mlp.experts.138.gate_proj", "model.layers.89.mlp.experts.139.gate_proj", "model.layers.89.mlp.experts.140.gate_proj", "model.layers.89.mlp.experts.141.gate_proj", "model.layers.89.mlp.experts.142.gate_proj", "model.layers.89.mlp.experts.143.gate_proj", "model.layers.89.mlp.experts.144.gate_proj", "model.layers.89.mlp.experts.145.gate_proj", "model.layers.89.mlp.experts.146.gate_proj", "model.layers.89.mlp.experts.147.gate_proj", "model.layers.89.mlp.experts.148.gate_proj", "model.layers.89.mlp.experts.149.gate_proj", "model.layers.89.mlp.experts.150.gate_proj", "model.layers.89.mlp.experts.151.gate_proj", "model.layers.89.mlp.experts.152.gate_proj", "model.layers.89.mlp.experts.153.gate_proj", "model.layers.89.mlp.experts.154.gate_proj", "model.layers.89.mlp.experts.155.gate_proj", "model.layers.89.mlp.experts.156.gate_proj", "model.layers.89.mlp.experts.157.gate_proj", "model.layers.89.mlp.experts.158.gate_proj", "model.layers.89.mlp.experts.159.gate_proj", "model.layers.89.mlp.experts.0.up_proj", "model.layers.89.mlp.experts.1.up_proj", "model.layers.89.mlp.experts.2.up_proj", "model.layers.89.mlp.experts.3.up_proj", "model.layers.89.mlp.experts.4.up_proj", "model.layers.89.mlp.experts.5.up_proj", "model.layers.89.mlp.experts.6.up_proj", "model.layers.89.mlp.experts.7.up_proj", "model.layers.89.mlp.experts.8.up_proj", "model.layers.89.mlp.experts.9.up_proj", "model.layers.89.mlp.experts.10.up_proj", "model.layers.89.mlp.experts.11.up_proj", "model.layers.89.mlp.experts.12.up_proj", "model.layers.89.mlp.experts.13.up_proj", "model.layers.89.mlp.experts.14.up_proj", "model.layers.89.mlp.experts.15.up_proj", "model.layers.89.mlp.experts.16.up_proj", "model.layers.89.mlp.experts.17.up_proj", "model.layers.89.mlp.experts.18.up_proj", "model.layers.89.mlp.experts.19.up_proj", "model.layers.89.mlp.experts.20.up_proj", "model.layers.89.mlp.experts.21.up_proj", "model.layers.89.mlp.experts.22.up_proj", "model.layers.89.mlp.experts.23.up_proj", "model.layers.89.mlp.experts.24.up_proj", "model.layers.89.mlp.experts.25.up_proj", "model.layers.89.mlp.experts.26.up_proj", "model.layers.89.mlp.experts.27.up_proj", "model.layers.89.mlp.experts.28.up_proj", "model.layers.89.mlp.experts.29.up_proj", "model.layers.89.mlp.experts.30.up_proj", "model.layers.89.mlp.experts.31.up_proj", "model.layers.89.mlp.experts.32.up_proj", "model.layers.89.mlp.experts.33.up_proj", "model.layers.89.mlp.experts.34.up_proj", "model.layers.89.mlp.experts.35.up_proj", "model.layers.89.mlp.experts.36.up_proj", "model.layers.89.mlp.experts.37.up_proj", "model.layers.89.mlp.experts.38.up_proj", "model.layers.89.mlp.experts.39.up_proj", "model.layers.89.mlp.experts.40.up_proj", "model.layers.89.mlp.experts.41.up_proj", "model.layers.89.mlp.experts.42.up_proj", "model.layers.89.mlp.experts.43.up_proj", "model.layers.89.mlp.experts.44.up_proj", "model.layers.89.mlp.experts.45.up_proj", "model.layers.89.mlp.experts.46.up_proj", "model.layers.89.mlp.experts.47.up_proj", "model.layers.89.mlp.experts.48.up_proj", "model.layers.89.mlp.experts.49.up_proj", "model.layers.89.mlp.experts.50.up_proj", "model.layers.89.mlp.experts.51.up_proj", "model.layers.89.mlp.experts.52.up_proj", "model.layers.89.mlp.experts.53.up_proj", "model.layers.89.mlp.experts.54.up_proj", "model.layers.89.mlp.experts.55.up_proj", "model.layers.89.mlp.experts.56.up_proj", "model.layers.89.mlp.experts.57.up_proj", "model.layers.89.mlp.experts.58.up_proj", "model.layers.89.mlp.experts.59.up_proj", "model.layers.89.mlp.experts.60.up_proj", "model.layers.89.mlp.experts.61.up_proj", "model.layers.89.mlp.experts.62.up_proj", "model.layers.89.mlp.experts.63.up_proj", "model.layers.89.mlp.experts.64.up_proj", "model.layers.89.mlp.experts.65.up_proj", "model.layers.89.mlp.experts.66.up_proj", "model.layers.89.mlp.experts.67.up_proj", "model.layers.89.mlp.experts.68.up_proj", "model.layers.89.mlp.experts.69.up_proj", "model.layers.89.mlp.experts.70.up_proj", "model.layers.89.mlp.experts.71.up_proj", "model.layers.89.mlp.experts.72.up_proj", "model.layers.89.mlp.experts.73.up_proj", "model.layers.89.mlp.experts.74.up_proj", "model.layers.89.mlp.experts.75.up_proj", "model.layers.89.mlp.experts.76.up_proj", "model.layers.89.mlp.experts.77.up_proj", "model.layers.89.mlp.experts.78.up_proj", "model.layers.89.mlp.experts.79.up_proj", "model.layers.89.mlp.experts.80.up_proj", "model.layers.89.mlp.experts.81.up_proj", "model.layers.89.mlp.experts.82.up_proj", "model.layers.89.mlp.experts.83.up_proj", "model.layers.89.mlp.experts.84.up_proj", "model.layers.89.mlp.experts.85.up_proj", "model.layers.89.mlp.experts.86.up_proj", "model.layers.89.mlp.experts.87.up_proj", "model.layers.89.mlp.experts.88.up_proj", "model.layers.89.mlp.experts.89.up_proj", "model.layers.89.mlp.experts.90.up_proj", "model.layers.89.mlp.experts.91.up_proj", "model.layers.89.mlp.experts.92.up_proj", "model.layers.89.mlp.experts.93.up_proj", "model.layers.89.mlp.experts.94.up_proj", "model.layers.89.mlp.experts.95.up_proj", "model.layers.89.mlp.experts.96.up_proj", "model.layers.89.mlp.experts.97.up_proj", "model.layers.89.mlp.experts.98.up_proj", "model.layers.89.mlp.experts.99.up_proj", "model.layers.89.mlp.experts.100.up_proj", "model.layers.89.mlp.experts.101.up_proj", "model.layers.89.mlp.experts.102.up_proj", "model.layers.89.mlp.experts.103.up_proj", "model.layers.89.mlp.experts.104.up_proj", "model.layers.89.mlp.experts.105.up_proj", "model.layers.89.mlp.experts.106.up_proj", "model.layers.89.mlp.experts.107.up_proj", "model.layers.89.mlp.experts.108.up_proj", "model.layers.89.mlp.experts.109.up_proj", "model.layers.89.mlp.experts.110.up_proj", "model.layers.89.mlp.experts.111.up_proj", "model.layers.89.mlp.experts.112.up_proj", "model.layers.89.mlp.experts.113.up_proj", "model.layers.89.mlp.experts.114.up_proj", "model.layers.89.mlp.experts.115.up_proj", "model.layers.89.mlp.experts.116.up_proj", "model.layers.89.mlp.experts.117.up_proj", "model.layers.89.mlp.experts.118.up_proj", "model.layers.89.mlp.experts.119.up_proj", "model.layers.89.mlp.experts.120.up_proj", "model.layers.89.mlp.experts.121.up_proj", "model.layers.89.mlp.experts.122.up_proj", "model.layers.89.mlp.experts.123.up_proj", "model.layers.89.mlp.experts.124.up_proj", "model.layers.89.mlp.experts.125.up_proj", "model.layers.89.mlp.experts.126.up_proj", "model.layers.89.mlp.experts.127.up_proj", "model.layers.89.mlp.experts.128.up_proj", "model.layers.89.mlp.experts.129.up_proj", "model.layers.89.mlp.experts.130.up_proj", "model.layers.89.mlp.experts.131.up_proj", "model.layers.89.mlp.experts.132.up_proj", "model.layers.89.mlp.experts.133.up_proj", "model.layers.89.mlp.experts.134.up_proj", "model.layers.89.mlp.experts.135.up_proj", "model.layers.89.mlp.experts.136.up_proj", "model.layers.89.mlp.experts.137.up_proj", "model.layers.89.mlp.experts.138.up_proj", "model.layers.89.mlp.experts.139.up_proj", "model.layers.89.mlp.experts.140.up_proj", "model.layers.89.mlp.experts.141.up_proj", "model.layers.89.mlp.experts.142.up_proj", "model.layers.89.mlp.experts.143.up_proj", "model.layers.89.mlp.experts.144.up_proj", "model.layers.89.mlp.experts.145.up_proj", "model.layers.89.mlp.experts.146.up_proj", "model.layers.89.mlp.experts.147.up_proj", "model.layers.89.mlp.experts.148.up_proj", "model.layers.89.mlp.experts.149.up_proj", "model.layers.89.mlp.experts.150.up_proj", "model.layers.89.mlp.experts.151.up_proj", "model.layers.89.mlp.experts.152.up_proj", "model.layers.89.mlp.experts.153.up_proj", "model.layers.89.mlp.experts.154.up_proj", "model.layers.89.mlp.experts.155.up_proj", "model.layers.89.mlp.experts.156.up_proj", "model.layers.89.mlp.experts.157.up_proj", "model.layers.89.mlp.experts.158.up_proj", "model.layers.89.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 7.151829777285457e-07, "dbits": 5033164800 } ] }, { "idx": 536, "layers": [ "model.layers.89.mlp.experts.0.down_proj", "model.layers.89.mlp.experts.1.down_proj", "model.layers.89.mlp.experts.2.down_proj", "model.layers.89.mlp.experts.3.down_proj", "model.layers.89.mlp.experts.4.down_proj", "model.layers.89.mlp.experts.5.down_proj", "model.layers.89.mlp.experts.6.down_proj", "model.layers.89.mlp.experts.7.down_proj", "model.layers.89.mlp.experts.8.down_proj", "model.layers.89.mlp.experts.9.down_proj", "model.layers.89.mlp.experts.10.down_proj", "model.layers.89.mlp.experts.11.down_proj", "model.layers.89.mlp.experts.12.down_proj", "model.layers.89.mlp.experts.13.down_proj", "model.layers.89.mlp.experts.14.down_proj", "model.layers.89.mlp.experts.15.down_proj", "model.layers.89.mlp.experts.16.down_proj", "model.layers.89.mlp.experts.17.down_proj", "model.layers.89.mlp.experts.18.down_proj", "model.layers.89.mlp.experts.19.down_proj", "model.layers.89.mlp.experts.20.down_proj", "model.layers.89.mlp.experts.21.down_proj", "model.layers.89.mlp.experts.22.down_proj", "model.layers.89.mlp.experts.23.down_proj", "model.layers.89.mlp.experts.24.down_proj", "model.layers.89.mlp.experts.25.down_proj", "model.layers.89.mlp.experts.26.down_proj", "model.layers.89.mlp.experts.27.down_proj", "model.layers.89.mlp.experts.28.down_proj", "model.layers.89.mlp.experts.29.down_proj", "model.layers.89.mlp.experts.30.down_proj", "model.layers.89.mlp.experts.31.down_proj", "model.layers.89.mlp.experts.32.down_proj", "model.layers.89.mlp.experts.33.down_proj", "model.layers.89.mlp.experts.34.down_proj", "model.layers.89.mlp.experts.35.down_proj", "model.layers.89.mlp.experts.36.down_proj", "model.layers.89.mlp.experts.37.down_proj", "model.layers.89.mlp.experts.38.down_proj", "model.layers.89.mlp.experts.39.down_proj", "model.layers.89.mlp.experts.40.down_proj", "model.layers.89.mlp.experts.41.down_proj", "model.layers.89.mlp.experts.42.down_proj", "model.layers.89.mlp.experts.43.down_proj", "model.layers.89.mlp.experts.44.down_proj", "model.layers.89.mlp.experts.45.down_proj", "model.layers.89.mlp.experts.46.down_proj", "model.layers.89.mlp.experts.47.down_proj", "model.layers.89.mlp.experts.48.down_proj", "model.layers.89.mlp.experts.49.down_proj", "model.layers.89.mlp.experts.50.down_proj", "model.layers.89.mlp.experts.51.down_proj", "model.layers.89.mlp.experts.52.down_proj", "model.layers.89.mlp.experts.53.down_proj", "model.layers.89.mlp.experts.54.down_proj", "model.layers.89.mlp.experts.55.down_proj", "model.layers.89.mlp.experts.56.down_proj", "model.layers.89.mlp.experts.57.down_proj", "model.layers.89.mlp.experts.58.down_proj", "model.layers.89.mlp.experts.59.down_proj", "model.layers.89.mlp.experts.60.down_proj", "model.layers.89.mlp.experts.61.down_proj", "model.layers.89.mlp.experts.62.down_proj", "model.layers.89.mlp.experts.63.down_proj", "model.layers.89.mlp.experts.64.down_proj", "model.layers.89.mlp.experts.65.down_proj", "model.layers.89.mlp.experts.66.down_proj", "model.layers.89.mlp.experts.67.down_proj", "model.layers.89.mlp.experts.68.down_proj", "model.layers.89.mlp.experts.69.down_proj", "model.layers.89.mlp.experts.70.down_proj", "model.layers.89.mlp.experts.71.down_proj", "model.layers.89.mlp.experts.72.down_proj", "model.layers.89.mlp.experts.73.down_proj", "model.layers.89.mlp.experts.74.down_proj", "model.layers.89.mlp.experts.75.down_proj", "model.layers.89.mlp.experts.76.down_proj", "model.layers.89.mlp.experts.77.down_proj", "model.layers.89.mlp.experts.78.down_proj", "model.layers.89.mlp.experts.79.down_proj", "model.layers.89.mlp.experts.80.down_proj", "model.layers.89.mlp.experts.81.down_proj", "model.layers.89.mlp.experts.82.down_proj", "model.layers.89.mlp.experts.83.down_proj", "model.layers.89.mlp.experts.84.down_proj", "model.layers.89.mlp.experts.85.down_proj", "model.layers.89.mlp.experts.86.down_proj", "model.layers.89.mlp.experts.87.down_proj", "model.layers.89.mlp.experts.88.down_proj", "model.layers.89.mlp.experts.89.down_proj", "model.layers.89.mlp.experts.90.down_proj", "model.layers.89.mlp.experts.91.down_proj", "model.layers.89.mlp.experts.92.down_proj", "model.layers.89.mlp.experts.93.down_proj", "model.layers.89.mlp.experts.94.down_proj", "model.layers.89.mlp.experts.95.down_proj", "model.layers.89.mlp.experts.96.down_proj", "model.layers.89.mlp.experts.97.down_proj", "model.layers.89.mlp.experts.98.down_proj", "model.layers.89.mlp.experts.99.down_proj", "model.layers.89.mlp.experts.100.down_proj", "model.layers.89.mlp.experts.101.down_proj", "model.layers.89.mlp.experts.102.down_proj", "model.layers.89.mlp.experts.103.down_proj", "model.layers.89.mlp.experts.104.down_proj", "model.layers.89.mlp.experts.105.down_proj", "model.layers.89.mlp.experts.106.down_proj", "model.layers.89.mlp.experts.107.down_proj", "model.layers.89.mlp.experts.108.down_proj", "model.layers.89.mlp.experts.109.down_proj", "model.layers.89.mlp.experts.110.down_proj", "model.layers.89.mlp.experts.111.down_proj", "model.layers.89.mlp.experts.112.down_proj", "model.layers.89.mlp.experts.113.down_proj", "model.layers.89.mlp.experts.114.down_proj", "model.layers.89.mlp.experts.115.down_proj", "model.layers.89.mlp.experts.116.down_proj", "model.layers.89.mlp.experts.117.down_proj", "model.layers.89.mlp.experts.118.down_proj", "model.layers.89.mlp.experts.119.down_proj", "model.layers.89.mlp.experts.120.down_proj", "model.layers.89.mlp.experts.121.down_proj", "model.layers.89.mlp.experts.122.down_proj", "model.layers.89.mlp.experts.123.down_proj", "model.layers.89.mlp.experts.124.down_proj", "model.layers.89.mlp.experts.125.down_proj", "model.layers.89.mlp.experts.126.down_proj", "model.layers.89.mlp.experts.127.down_proj", "model.layers.89.mlp.experts.128.down_proj", "model.layers.89.mlp.experts.129.down_proj", "model.layers.89.mlp.experts.130.down_proj", "model.layers.89.mlp.experts.131.down_proj", "model.layers.89.mlp.experts.132.down_proj", "model.layers.89.mlp.experts.133.down_proj", "model.layers.89.mlp.experts.134.down_proj", "model.layers.89.mlp.experts.135.down_proj", "model.layers.89.mlp.experts.136.down_proj", "model.layers.89.mlp.experts.137.down_proj", "model.layers.89.mlp.experts.138.down_proj", "model.layers.89.mlp.experts.139.down_proj", "model.layers.89.mlp.experts.140.down_proj", "model.layers.89.mlp.experts.141.down_proj", "model.layers.89.mlp.experts.142.down_proj", "model.layers.89.mlp.experts.143.down_proj", "model.layers.89.mlp.experts.144.down_proj", "model.layers.89.mlp.experts.145.down_proj", "model.layers.89.mlp.experts.146.down_proj", "model.layers.89.mlp.experts.147.down_proj", "model.layers.89.mlp.experts.148.down_proj", "model.layers.89.mlp.experts.149.down_proj", "model.layers.89.mlp.experts.150.down_proj", "model.layers.89.mlp.experts.151.down_proj", "model.layers.89.mlp.experts.152.down_proj", "model.layers.89.mlp.experts.153.down_proj", "model.layers.89.mlp.experts.154.down_proj", "model.layers.89.mlp.experts.155.down_proj", "model.layers.89.mlp.experts.156.down_proj", "model.layers.89.mlp.experts.157.down_proj", "model.layers.89.mlp.experts.158.down_proj", "model.layers.89.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 1.6974023310467818e-06, "dbits": 2516582400 } ] }, { "idx": 537, "layers": [ "model.layers.90.self_attn.q_proj" ], "candidates": [ { "dkld": 2.101997961290224e-06, "dbits": 125829120 } ] }, { "idx": 538, "layers": [ "model.layers.90.self_attn.k_proj", "model.layers.90.self_attn.v_proj" ], "candidates": [ { "dkld": -1.2010888895020323e-06, "dbits": 20971520 } ] }, { "idx": 539, "layers": [ "model.layers.90.self_attn.o_proj" ], "candidates": [ { "dkld": -8.224538760259964e-07, "dbits": 125829120 } ] }, { "idx": 540, "layers": [ "model.layers.90.mlp.shared_experts.gate_proj", "model.layers.90.mlp.shared_experts.up_proj", "model.layers.90.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.181802018545611e-06, "dbits": 47185920 } ] }, { "idx": 541, "layers": [ "model.layers.90.mlp.experts.0.gate_proj", "model.layers.90.mlp.experts.1.gate_proj", "model.layers.90.mlp.experts.2.gate_proj", "model.layers.90.mlp.experts.3.gate_proj", "model.layers.90.mlp.experts.4.gate_proj", "model.layers.90.mlp.experts.5.gate_proj", "model.layers.90.mlp.experts.6.gate_proj", "model.layers.90.mlp.experts.7.gate_proj", "model.layers.90.mlp.experts.8.gate_proj", "model.layers.90.mlp.experts.9.gate_proj", "model.layers.90.mlp.experts.10.gate_proj", "model.layers.90.mlp.experts.11.gate_proj", "model.layers.90.mlp.experts.12.gate_proj", "model.layers.90.mlp.experts.13.gate_proj", "model.layers.90.mlp.experts.14.gate_proj", "model.layers.90.mlp.experts.15.gate_proj", "model.layers.90.mlp.experts.16.gate_proj", "model.layers.90.mlp.experts.17.gate_proj", "model.layers.90.mlp.experts.18.gate_proj", "model.layers.90.mlp.experts.19.gate_proj", "model.layers.90.mlp.experts.20.gate_proj", "model.layers.90.mlp.experts.21.gate_proj", "model.layers.90.mlp.experts.22.gate_proj", "model.layers.90.mlp.experts.23.gate_proj", "model.layers.90.mlp.experts.24.gate_proj", "model.layers.90.mlp.experts.25.gate_proj", "model.layers.90.mlp.experts.26.gate_proj", "model.layers.90.mlp.experts.27.gate_proj", "model.layers.90.mlp.experts.28.gate_proj", "model.layers.90.mlp.experts.29.gate_proj", "model.layers.90.mlp.experts.30.gate_proj", "model.layers.90.mlp.experts.31.gate_proj", "model.layers.90.mlp.experts.32.gate_proj", "model.layers.90.mlp.experts.33.gate_proj", "model.layers.90.mlp.experts.34.gate_proj", "model.layers.90.mlp.experts.35.gate_proj", "model.layers.90.mlp.experts.36.gate_proj", "model.layers.90.mlp.experts.37.gate_proj", "model.layers.90.mlp.experts.38.gate_proj", "model.layers.90.mlp.experts.39.gate_proj", "model.layers.90.mlp.experts.40.gate_proj", "model.layers.90.mlp.experts.41.gate_proj", "model.layers.90.mlp.experts.42.gate_proj", "model.layers.90.mlp.experts.43.gate_proj", "model.layers.90.mlp.experts.44.gate_proj", "model.layers.90.mlp.experts.45.gate_proj", "model.layers.90.mlp.experts.46.gate_proj", "model.layers.90.mlp.experts.47.gate_proj", "model.layers.90.mlp.experts.48.gate_proj", "model.layers.90.mlp.experts.49.gate_proj", "model.layers.90.mlp.experts.50.gate_proj", "model.layers.90.mlp.experts.51.gate_proj", "model.layers.90.mlp.experts.52.gate_proj", "model.layers.90.mlp.experts.53.gate_proj", "model.layers.90.mlp.experts.54.gate_proj", "model.layers.90.mlp.experts.55.gate_proj", "model.layers.90.mlp.experts.56.gate_proj", "model.layers.90.mlp.experts.57.gate_proj", "model.layers.90.mlp.experts.58.gate_proj", "model.layers.90.mlp.experts.59.gate_proj", "model.layers.90.mlp.experts.60.gate_proj", "model.layers.90.mlp.experts.61.gate_proj", "model.layers.90.mlp.experts.62.gate_proj", "model.layers.90.mlp.experts.63.gate_proj", "model.layers.90.mlp.experts.64.gate_proj", "model.layers.90.mlp.experts.65.gate_proj", "model.layers.90.mlp.experts.66.gate_proj", "model.layers.90.mlp.experts.67.gate_proj", "model.layers.90.mlp.experts.68.gate_proj", "model.layers.90.mlp.experts.69.gate_proj", "model.layers.90.mlp.experts.70.gate_proj", "model.layers.90.mlp.experts.71.gate_proj", "model.layers.90.mlp.experts.72.gate_proj", "model.layers.90.mlp.experts.73.gate_proj", "model.layers.90.mlp.experts.74.gate_proj", "model.layers.90.mlp.experts.75.gate_proj", "model.layers.90.mlp.experts.76.gate_proj", "model.layers.90.mlp.experts.77.gate_proj", "model.layers.90.mlp.experts.78.gate_proj", "model.layers.90.mlp.experts.79.gate_proj", "model.layers.90.mlp.experts.80.gate_proj", "model.layers.90.mlp.experts.81.gate_proj", "model.layers.90.mlp.experts.82.gate_proj", "model.layers.90.mlp.experts.83.gate_proj", "model.layers.90.mlp.experts.84.gate_proj", "model.layers.90.mlp.experts.85.gate_proj", "model.layers.90.mlp.experts.86.gate_proj", "model.layers.90.mlp.experts.87.gate_proj", "model.layers.90.mlp.experts.88.gate_proj", "model.layers.90.mlp.experts.89.gate_proj", "model.layers.90.mlp.experts.90.gate_proj", "model.layers.90.mlp.experts.91.gate_proj", "model.layers.90.mlp.experts.92.gate_proj", "model.layers.90.mlp.experts.93.gate_proj", "model.layers.90.mlp.experts.94.gate_proj", "model.layers.90.mlp.experts.95.gate_proj", "model.layers.90.mlp.experts.96.gate_proj", "model.layers.90.mlp.experts.97.gate_proj", "model.layers.90.mlp.experts.98.gate_proj", "model.layers.90.mlp.experts.99.gate_proj", "model.layers.90.mlp.experts.100.gate_proj", "model.layers.90.mlp.experts.101.gate_proj", "model.layers.90.mlp.experts.102.gate_proj", "model.layers.90.mlp.experts.103.gate_proj", "model.layers.90.mlp.experts.104.gate_proj", "model.layers.90.mlp.experts.105.gate_proj", "model.layers.90.mlp.experts.106.gate_proj", "model.layers.90.mlp.experts.107.gate_proj", "model.layers.90.mlp.experts.108.gate_proj", "model.layers.90.mlp.experts.109.gate_proj", "model.layers.90.mlp.experts.110.gate_proj", "model.layers.90.mlp.experts.111.gate_proj", "model.layers.90.mlp.experts.112.gate_proj", "model.layers.90.mlp.experts.113.gate_proj", "model.layers.90.mlp.experts.114.gate_proj", "model.layers.90.mlp.experts.115.gate_proj", "model.layers.90.mlp.experts.116.gate_proj", "model.layers.90.mlp.experts.117.gate_proj", "model.layers.90.mlp.experts.118.gate_proj", "model.layers.90.mlp.experts.119.gate_proj", "model.layers.90.mlp.experts.120.gate_proj", "model.layers.90.mlp.experts.121.gate_proj", "model.layers.90.mlp.experts.122.gate_proj", "model.layers.90.mlp.experts.123.gate_proj", "model.layers.90.mlp.experts.124.gate_proj", "model.layers.90.mlp.experts.125.gate_proj", "model.layers.90.mlp.experts.126.gate_proj", "model.layers.90.mlp.experts.127.gate_proj", "model.layers.90.mlp.experts.128.gate_proj", "model.layers.90.mlp.experts.129.gate_proj", "model.layers.90.mlp.experts.130.gate_proj", "model.layers.90.mlp.experts.131.gate_proj", "model.layers.90.mlp.experts.132.gate_proj", "model.layers.90.mlp.experts.133.gate_proj", "model.layers.90.mlp.experts.134.gate_proj", "model.layers.90.mlp.experts.135.gate_proj", "model.layers.90.mlp.experts.136.gate_proj", "model.layers.90.mlp.experts.137.gate_proj", "model.layers.90.mlp.experts.138.gate_proj", "model.layers.90.mlp.experts.139.gate_proj", "model.layers.90.mlp.experts.140.gate_proj", "model.layers.90.mlp.experts.141.gate_proj", "model.layers.90.mlp.experts.142.gate_proj", "model.layers.90.mlp.experts.143.gate_proj", "model.layers.90.mlp.experts.144.gate_proj", "model.layers.90.mlp.experts.145.gate_proj", "model.layers.90.mlp.experts.146.gate_proj", "model.layers.90.mlp.experts.147.gate_proj", "model.layers.90.mlp.experts.148.gate_proj", "model.layers.90.mlp.experts.149.gate_proj", "model.layers.90.mlp.experts.150.gate_proj", "model.layers.90.mlp.experts.151.gate_proj", "model.layers.90.mlp.experts.152.gate_proj", "model.layers.90.mlp.experts.153.gate_proj", "model.layers.90.mlp.experts.154.gate_proj", "model.layers.90.mlp.experts.155.gate_proj", "model.layers.90.mlp.experts.156.gate_proj", "model.layers.90.mlp.experts.157.gate_proj", "model.layers.90.mlp.experts.158.gate_proj", "model.layers.90.mlp.experts.159.gate_proj", "model.layers.90.mlp.experts.0.up_proj", "model.layers.90.mlp.experts.1.up_proj", "model.layers.90.mlp.experts.2.up_proj", "model.layers.90.mlp.experts.3.up_proj", "model.layers.90.mlp.experts.4.up_proj", "model.layers.90.mlp.experts.5.up_proj", "model.layers.90.mlp.experts.6.up_proj", "model.layers.90.mlp.experts.7.up_proj", "model.layers.90.mlp.experts.8.up_proj", "model.layers.90.mlp.experts.9.up_proj", "model.layers.90.mlp.experts.10.up_proj", "model.layers.90.mlp.experts.11.up_proj", "model.layers.90.mlp.experts.12.up_proj", "model.layers.90.mlp.experts.13.up_proj", "model.layers.90.mlp.experts.14.up_proj", "model.layers.90.mlp.experts.15.up_proj", "model.layers.90.mlp.experts.16.up_proj", "model.layers.90.mlp.experts.17.up_proj", "model.layers.90.mlp.experts.18.up_proj", "model.layers.90.mlp.experts.19.up_proj", "model.layers.90.mlp.experts.20.up_proj", "model.layers.90.mlp.experts.21.up_proj", "model.layers.90.mlp.experts.22.up_proj", "model.layers.90.mlp.experts.23.up_proj", "model.layers.90.mlp.experts.24.up_proj", "model.layers.90.mlp.experts.25.up_proj", "model.layers.90.mlp.experts.26.up_proj", "model.layers.90.mlp.experts.27.up_proj", "model.layers.90.mlp.experts.28.up_proj", "model.layers.90.mlp.experts.29.up_proj", "model.layers.90.mlp.experts.30.up_proj", "model.layers.90.mlp.experts.31.up_proj", "model.layers.90.mlp.experts.32.up_proj", "model.layers.90.mlp.experts.33.up_proj", "model.layers.90.mlp.experts.34.up_proj", "model.layers.90.mlp.experts.35.up_proj", "model.layers.90.mlp.experts.36.up_proj", "model.layers.90.mlp.experts.37.up_proj", "model.layers.90.mlp.experts.38.up_proj", "model.layers.90.mlp.experts.39.up_proj", "model.layers.90.mlp.experts.40.up_proj", "model.layers.90.mlp.experts.41.up_proj", "model.layers.90.mlp.experts.42.up_proj", "model.layers.90.mlp.experts.43.up_proj", "model.layers.90.mlp.experts.44.up_proj", "model.layers.90.mlp.experts.45.up_proj", "model.layers.90.mlp.experts.46.up_proj", "model.layers.90.mlp.experts.47.up_proj", "model.layers.90.mlp.experts.48.up_proj", "model.layers.90.mlp.experts.49.up_proj", "model.layers.90.mlp.experts.50.up_proj", "model.layers.90.mlp.experts.51.up_proj", "model.layers.90.mlp.experts.52.up_proj", "model.layers.90.mlp.experts.53.up_proj", "model.layers.90.mlp.experts.54.up_proj", "model.layers.90.mlp.experts.55.up_proj", "model.layers.90.mlp.experts.56.up_proj", "model.layers.90.mlp.experts.57.up_proj", "model.layers.90.mlp.experts.58.up_proj", "model.layers.90.mlp.experts.59.up_proj", "model.layers.90.mlp.experts.60.up_proj", "model.layers.90.mlp.experts.61.up_proj", "model.layers.90.mlp.experts.62.up_proj", "model.layers.90.mlp.experts.63.up_proj", "model.layers.90.mlp.experts.64.up_proj", "model.layers.90.mlp.experts.65.up_proj", "model.layers.90.mlp.experts.66.up_proj", "model.layers.90.mlp.experts.67.up_proj", "model.layers.90.mlp.experts.68.up_proj", "model.layers.90.mlp.experts.69.up_proj", "model.layers.90.mlp.experts.70.up_proj", "model.layers.90.mlp.experts.71.up_proj", "model.layers.90.mlp.experts.72.up_proj", "model.layers.90.mlp.experts.73.up_proj", "model.layers.90.mlp.experts.74.up_proj", "model.layers.90.mlp.experts.75.up_proj", "model.layers.90.mlp.experts.76.up_proj", "model.layers.90.mlp.experts.77.up_proj", "model.layers.90.mlp.experts.78.up_proj", "model.layers.90.mlp.experts.79.up_proj", "model.layers.90.mlp.experts.80.up_proj", "model.layers.90.mlp.experts.81.up_proj", "model.layers.90.mlp.experts.82.up_proj", "model.layers.90.mlp.experts.83.up_proj", "model.layers.90.mlp.experts.84.up_proj", "model.layers.90.mlp.experts.85.up_proj", "model.layers.90.mlp.experts.86.up_proj", "model.layers.90.mlp.experts.87.up_proj", "model.layers.90.mlp.experts.88.up_proj", "model.layers.90.mlp.experts.89.up_proj", "model.layers.90.mlp.experts.90.up_proj", "model.layers.90.mlp.experts.91.up_proj", "model.layers.90.mlp.experts.92.up_proj", "model.layers.90.mlp.experts.93.up_proj", "model.layers.90.mlp.experts.94.up_proj", "model.layers.90.mlp.experts.95.up_proj", "model.layers.90.mlp.experts.96.up_proj", "model.layers.90.mlp.experts.97.up_proj", "model.layers.90.mlp.experts.98.up_proj", "model.layers.90.mlp.experts.99.up_proj", "model.layers.90.mlp.experts.100.up_proj", "model.layers.90.mlp.experts.101.up_proj", "model.layers.90.mlp.experts.102.up_proj", "model.layers.90.mlp.experts.103.up_proj", "model.layers.90.mlp.experts.104.up_proj", "model.layers.90.mlp.experts.105.up_proj", "model.layers.90.mlp.experts.106.up_proj", "model.layers.90.mlp.experts.107.up_proj", "model.layers.90.mlp.experts.108.up_proj", "model.layers.90.mlp.experts.109.up_proj", "model.layers.90.mlp.experts.110.up_proj", "model.layers.90.mlp.experts.111.up_proj", "model.layers.90.mlp.experts.112.up_proj", "model.layers.90.mlp.experts.113.up_proj", "model.layers.90.mlp.experts.114.up_proj", "model.layers.90.mlp.experts.115.up_proj", "model.layers.90.mlp.experts.116.up_proj", "model.layers.90.mlp.experts.117.up_proj", "model.layers.90.mlp.experts.118.up_proj", "model.layers.90.mlp.experts.119.up_proj", "model.layers.90.mlp.experts.120.up_proj", "model.layers.90.mlp.experts.121.up_proj", "model.layers.90.mlp.experts.122.up_proj", "model.layers.90.mlp.experts.123.up_proj", "model.layers.90.mlp.experts.124.up_proj", "model.layers.90.mlp.experts.125.up_proj", "model.layers.90.mlp.experts.126.up_proj", "model.layers.90.mlp.experts.127.up_proj", "model.layers.90.mlp.experts.128.up_proj", "model.layers.90.mlp.experts.129.up_proj", "model.layers.90.mlp.experts.130.up_proj", "model.layers.90.mlp.experts.131.up_proj", "model.layers.90.mlp.experts.132.up_proj", "model.layers.90.mlp.experts.133.up_proj", "model.layers.90.mlp.experts.134.up_proj", "model.layers.90.mlp.experts.135.up_proj", "model.layers.90.mlp.experts.136.up_proj", "model.layers.90.mlp.experts.137.up_proj", "model.layers.90.mlp.experts.138.up_proj", "model.layers.90.mlp.experts.139.up_proj", "model.layers.90.mlp.experts.140.up_proj", "model.layers.90.mlp.experts.141.up_proj", "model.layers.90.mlp.experts.142.up_proj", "model.layers.90.mlp.experts.143.up_proj", "model.layers.90.mlp.experts.144.up_proj", "model.layers.90.mlp.experts.145.up_proj", "model.layers.90.mlp.experts.146.up_proj", "model.layers.90.mlp.experts.147.up_proj", "model.layers.90.mlp.experts.148.up_proj", "model.layers.90.mlp.experts.149.up_proj", "model.layers.90.mlp.experts.150.up_proj", "model.layers.90.mlp.experts.151.up_proj", "model.layers.90.mlp.experts.152.up_proj", "model.layers.90.mlp.experts.153.up_proj", "model.layers.90.mlp.experts.154.up_proj", "model.layers.90.mlp.experts.155.up_proj", "model.layers.90.mlp.experts.156.up_proj", "model.layers.90.mlp.experts.157.up_proj", "model.layers.90.mlp.experts.158.up_proj", "model.layers.90.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 9.172945283353329e-07, "dbits": 5033164800 } ] }, { "idx": 542, "layers": [ "model.layers.90.mlp.experts.0.down_proj", "model.layers.90.mlp.experts.1.down_proj", "model.layers.90.mlp.experts.2.down_proj", "model.layers.90.mlp.experts.3.down_proj", "model.layers.90.mlp.experts.4.down_proj", "model.layers.90.mlp.experts.5.down_proj", "model.layers.90.mlp.experts.6.down_proj", "model.layers.90.mlp.experts.7.down_proj", "model.layers.90.mlp.experts.8.down_proj", "model.layers.90.mlp.experts.9.down_proj", "model.layers.90.mlp.experts.10.down_proj", "model.layers.90.mlp.experts.11.down_proj", "model.layers.90.mlp.experts.12.down_proj", "model.layers.90.mlp.experts.13.down_proj", "model.layers.90.mlp.experts.14.down_proj", "model.layers.90.mlp.experts.15.down_proj", "model.layers.90.mlp.experts.16.down_proj", "model.layers.90.mlp.experts.17.down_proj", "model.layers.90.mlp.experts.18.down_proj", "model.layers.90.mlp.experts.19.down_proj", "model.layers.90.mlp.experts.20.down_proj", "model.layers.90.mlp.experts.21.down_proj", "model.layers.90.mlp.experts.22.down_proj", "model.layers.90.mlp.experts.23.down_proj", "model.layers.90.mlp.experts.24.down_proj", "model.layers.90.mlp.experts.25.down_proj", "model.layers.90.mlp.experts.26.down_proj", "model.layers.90.mlp.experts.27.down_proj", "model.layers.90.mlp.experts.28.down_proj", "model.layers.90.mlp.experts.29.down_proj", "model.layers.90.mlp.experts.30.down_proj", "model.layers.90.mlp.experts.31.down_proj", "model.layers.90.mlp.experts.32.down_proj", "model.layers.90.mlp.experts.33.down_proj", "model.layers.90.mlp.experts.34.down_proj", "model.layers.90.mlp.experts.35.down_proj", "model.layers.90.mlp.experts.36.down_proj", "model.layers.90.mlp.experts.37.down_proj", "model.layers.90.mlp.experts.38.down_proj", "model.layers.90.mlp.experts.39.down_proj", "model.layers.90.mlp.experts.40.down_proj", "model.layers.90.mlp.experts.41.down_proj", "model.layers.90.mlp.experts.42.down_proj", "model.layers.90.mlp.experts.43.down_proj", "model.layers.90.mlp.experts.44.down_proj", "model.layers.90.mlp.experts.45.down_proj", "model.layers.90.mlp.experts.46.down_proj", "model.layers.90.mlp.experts.47.down_proj", "model.layers.90.mlp.experts.48.down_proj", "model.layers.90.mlp.experts.49.down_proj", "model.layers.90.mlp.experts.50.down_proj", "model.layers.90.mlp.experts.51.down_proj", "model.layers.90.mlp.experts.52.down_proj", "model.layers.90.mlp.experts.53.down_proj", "model.layers.90.mlp.experts.54.down_proj", "model.layers.90.mlp.experts.55.down_proj", "model.layers.90.mlp.experts.56.down_proj", "model.layers.90.mlp.experts.57.down_proj", "model.layers.90.mlp.experts.58.down_proj", "model.layers.90.mlp.experts.59.down_proj", "model.layers.90.mlp.experts.60.down_proj", "model.layers.90.mlp.experts.61.down_proj", "model.layers.90.mlp.experts.62.down_proj", "model.layers.90.mlp.experts.63.down_proj", "model.layers.90.mlp.experts.64.down_proj", "model.layers.90.mlp.experts.65.down_proj", "model.layers.90.mlp.experts.66.down_proj", "model.layers.90.mlp.experts.67.down_proj", "model.layers.90.mlp.experts.68.down_proj", "model.layers.90.mlp.experts.69.down_proj", "model.layers.90.mlp.experts.70.down_proj", "model.layers.90.mlp.experts.71.down_proj", "model.layers.90.mlp.experts.72.down_proj", "model.layers.90.mlp.experts.73.down_proj", "model.layers.90.mlp.experts.74.down_proj", "model.layers.90.mlp.experts.75.down_proj", "model.layers.90.mlp.experts.76.down_proj", "model.layers.90.mlp.experts.77.down_proj", "model.layers.90.mlp.experts.78.down_proj", "model.layers.90.mlp.experts.79.down_proj", "model.layers.90.mlp.experts.80.down_proj", "model.layers.90.mlp.experts.81.down_proj", "model.layers.90.mlp.experts.82.down_proj", "model.layers.90.mlp.experts.83.down_proj", "model.layers.90.mlp.experts.84.down_proj", "model.layers.90.mlp.experts.85.down_proj", "model.layers.90.mlp.experts.86.down_proj", "model.layers.90.mlp.experts.87.down_proj", "model.layers.90.mlp.experts.88.down_proj", "model.layers.90.mlp.experts.89.down_proj", "model.layers.90.mlp.experts.90.down_proj", "model.layers.90.mlp.experts.91.down_proj", "model.layers.90.mlp.experts.92.down_proj", "model.layers.90.mlp.experts.93.down_proj", "model.layers.90.mlp.experts.94.down_proj", "model.layers.90.mlp.experts.95.down_proj", "model.layers.90.mlp.experts.96.down_proj", "model.layers.90.mlp.experts.97.down_proj", "model.layers.90.mlp.experts.98.down_proj", "model.layers.90.mlp.experts.99.down_proj", "model.layers.90.mlp.experts.100.down_proj", "model.layers.90.mlp.experts.101.down_proj", "model.layers.90.mlp.experts.102.down_proj", "model.layers.90.mlp.experts.103.down_proj", "model.layers.90.mlp.experts.104.down_proj", "model.layers.90.mlp.experts.105.down_proj", "model.layers.90.mlp.experts.106.down_proj", "model.layers.90.mlp.experts.107.down_proj", "model.layers.90.mlp.experts.108.down_proj", "model.layers.90.mlp.experts.109.down_proj", "model.layers.90.mlp.experts.110.down_proj", "model.layers.90.mlp.experts.111.down_proj", "model.layers.90.mlp.experts.112.down_proj", "model.layers.90.mlp.experts.113.down_proj", "model.layers.90.mlp.experts.114.down_proj", "model.layers.90.mlp.experts.115.down_proj", "model.layers.90.mlp.experts.116.down_proj", "model.layers.90.mlp.experts.117.down_proj", "model.layers.90.mlp.experts.118.down_proj", "model.layers.90.mlp.experts.119.down_proj", "model.layers.90.mlp.experts.120.down_proj", "model.layers.90.mlp.experts.121.down_proj", "model.layers.90.mlp.experts.122.down_proj", "model.layers.90.mlp.experts.123.down_proj", "model.layers.90.mlp.experts.124.down_proj", "model.layers.90.mlp.experts.125.down_proj", "model.layers.90.mlp.experts.126.down_proj", "model.layers.90.mlp.experts.127.down_proj", "model.layers.90.mlp.experts.128.down_proj", "model.layers.90.mlp.experts.129.down_proj", "model.layers.90.mlp.experts.130.down_proj", "model.layers.90.mlp.experts.131.down_proj", "model.layers.90.mlp.experts.132.down_proj", "model.layers.90.mlp.experts.133.down_proj", "model.layers.90.mlp.experts.134.down_proj", "model.layers.90.mlp.experts.135.down_proj", "model.layers.90.mlp.experts.136.down_proj", "model.layers.90.mlp.experts.137.down_proj", "model.layers.90.mlp.experts.138.down_proj", "model.layers.90.mlp.experts.139.down_proj", "model.layers.90.mlp.experts.140.down_proj", "model.layers.90.mlp.experts.141.down_proj", "model.layers.90.mlp.experts.142.down_proj", "model.layers.90.mlp.experts.143.down_proj", "model.layers.90.mlp.experts.144.down_proj", "model.layers.90.mlp.experts.145.down_proj", "model.layers.90.mlp.experts.146.down_proj", "model.layers.90.mlp.experts.147.down_proj", "model.layers.90.mlp.experts.148.down_proj", "model.layers.90.mlp.experts.149.down_proj", "model.layers.90.mlp.experts.150.down_proj", "model.layers.90.mlp.experts.151.down_proj", "model.layers.90.mlp.experts.152.down_proj", "model.layers.90.mlp.experts.153.down_proj", "model.layers.90.mlp.experts.154.down_proj", "model.layers.90.mlp.experts.155.down_proj", "model.layers.90.mlp.experts.156.down_proj", "model.layers.90.mlp.experts.157.down_proj", "model.layers.90.mlp.experts.158.down_proj", "model.layers.90.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 4.883739165961309e-07, "dbits": 2516582400 } ] }, { "idx": 543, "layers": [ "model.layers.91.self_attn.q_proj" ], "candidates": [ { "dkld": 2.2471300326287963e-06, "dbits": 125829120 } ] }, { "idx": 544, "layers": [ "model.layers.91.self_attn.k_proj", "model.layers.91.self_attn.v_proj" ], "candidates": [ { "dkld": -3.6476820241659446e-06, "dbits": 20971520 } ] }, { "idx": 545, "layers": [ "model.layers.91.self_attn.o_proj" ], "candidates": [ { "dkld": -1.3245095033198812e-06, "dbits": 125829120 } ] }, { "idx": 546, "layers": [ "model.layers.91.mlp.shared_experts.gate_proj", "model.layers.91.mlp.shared_experts.up_proj", "model.layers.91.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.180546289310076e-07, "dbits": 47185920 } ] }, { "idx": 547, "layers": [ "model.layers.91.mlp.experts.0.gate_proj", "model.layers.91.mlp.experts.1.gate_proj", "model.layers.91.mlp.experts.2.gate_proj", "model.layers.91.mlp.experts.3.gate_proj", "model.layers.91.mlp.experts.4.gate_proj", "model.layers.91.mlp.experts.5.gate_proj", "model.layers.91.mlp.experts.6.gate_proj", "model.layers.91.mlp.experts.7.gate_proj", "model.layers.91.mlp.experts.8.gate_proj", "model.layers.91.mlp.experts.9.gate_proj", "model.layers.91.mlp.experts.10.gate_proj", "model.layers.91.mlp.experts.11.gate_proj", "model.layers.91.mlp.experts.12.gate_proj", "model.layers.91.mlp.experts.13.gate_proj", "model.layers.91.mlp.experts.14.gate_proj", "model.layers.91.mlp.experts.15.gate_proj", "model.layers.91.mlp.experts.16.gate_proj", "model.layers.91.mlp.experts.17.gate_proj", "model.layers.91.mlp.experts.18.gate_proj", "model.layers.91.mlp.experts.19.gate_proj", "model.layers.91.mlp.experts.20.gate_proj", "model.layers.91.mlp.experts.21.gate_proj", "model.layers.91.mlp.experts.22.gate_proj", "model.layers.91.mlp.experts.23.gate_proj", "model.layers.91.mlp.experts.24.gate_proj", "model.layers.91.mlp.experts.25.gate_proj", "model.layers.91.mlp.experts.26.gate_proj", "model.layers.91.mlp.experts.27.gate_proj", "model.layers.91.mlp.experts.28.gate_proj", "model.layers.91.mlp.experts.29.gate_proj", "model.layers.91.mlp.experts.30.gate_proj", "model.layers.91.mlp.experts.31.gate_proj", "model.layers.91.mlp.experts.32.gate_proj", "model.layers.91.mlp.experts.33.gate_proj", "model.layers.91.mlp.experts.34.gate_proj", "model.layers.91.mlp.experts.35.gate_proj", "model.layers.91.mlp.experts.36.gate_proj", "model.layers.91.mlp.experts.37.gate_proj", "model.layers.91.mlp.experts.38.gate_proj", "model.layers.91.mlp.experts.39.gate_proj", "model.layers.91.mlp.experts.40.gate_proj", "model.layers.91.mlp.experts.41.gate_proj", "model.layers.91.mlp.experts.42.gate_proj", "model.layers.91.mlp.experts.43.gate_proj", "model.layers.91.mlp.experts.44.gate_proj", "model.layers.91.mlp.experts.45.gate_proj", "model.layers.91.mlp.experts.46.gate_proj", "model.layers.91.mlp.experts.47.gate_proj", "model.layers.91.mlp.experts.48.gate_proj", "model.layers.91.mlp.experts.49.gate_proj", "model.layers.91.mlp.experts.50.gate_proj", "model.layers.91.mlp.experts.51.gate_proj", "model.layers.91.mlp.experts.52.gate_proj", "model.layers.91.mlp.experts.53.gate_proj", "model.layers.91.mlp.experts.54.gate_proj", "model.layers.91.mlp.experts.55.gate_proj", "model.layers.91.mlp.experts.56.gate_proj", "model.layers.91.mlp.experts.57.gate_proj", "model.layers.91.mlp.experts.58.gate_proj", "model.layers.91.mlp.experts.59.gate_proj", "model.layers.91.mlp.experts.60.gate_proj", "model.layers.91.mlp.experts.61.gate_proj", "model.layers.91.mlp.experts.62.gate_proj", "model.layers.91.mlp.experts.63.gate_proj", "model.layers.91.mlp.experts.64.gate_proj", "model.layers.91.mlp.experts.65.gate_proj", "model.layers.91.mlp.experts.66.gate_proj", "model.layers.91.mlp.experts.67.gate_proj", "model.layers.91.mlp.experts.68.gate_proj", "model.layers.91.mlp.experts.69.gate_proj", "model.layers.91.mlp.experts.70.gate_proj", "model.layers.91.mlp.experts.71.gate_proj", "model.layers.91.mlp.experts.72.gate_proj", "model.layers.91.mlp.experts.73.gate_proj", "model.layers.91.mlp.experts.74.gate_proj", "model.layers.91.mlp.experts.75.gate_proj", "model.layers.91.mlp.experts.76.gate_proj", "model.layers.91.mlp.experts.77.gate_proj", "model.layers.91.mlp.experts.78.gate_proj", "model.layers.91.mlp.experts.79.gate_proj", "model.layers.91.mlp.experts.80.gate_proj", "model.layers.91.mlp.experts.81.gate_proj", "model.layers.91.mlp.experts.82.gate_proj", "model.layers.91.mlp.experts.83.gate_proj", "model.layers.91.mlp.experts.84.gate_proj", "model.layers.91.mlp.experts.85.gate_proj", "model.layers.91.mlp.experts.86.gate_proj", "model.layers.91.mlp.experts.87.gate_proj", "model.layers.91.mlp.experts.88.gate_proj", "model.layers.91.mlp.experts.89.gate_proj", "model.layers.91.mlp.experts.90.gate_proj", "model.layers.91.mlp.experts.91.gate_proj", "model.layers.91.mlp.experts.92.gate_proj", "model.layers.91.mlp.experts.93.gate_proj", "model.layers.91.mlp.experts.94.gate_proj", "model.layers.91.mlp.experts.95.gate_proj", "model.layers.91.mlp.experts.96.gate_proj", "model.layers.91.mlp.experts.97.gate_proj", "model.layers.91.mlp.experts.98.gate_proj", "model.layers.91.mlp.experts.99.gate_proj", "model.layers.91.mlp.experts.100.gate_proj", "model.layers.91.mlp.experts.101.gate_proj", "model.layers.91.mlp.experts.102.gate_proj", "model.layers.91.mlp.experts.103.gate_proj", "model.layers.91.mlp.experts.104.gate_proj", "model.layers.91.mlp.experts.105.gate_proj", "model.layers.91.mlp.experts.106.gate_proj", "model.layers.91.mlp.experts.107.gate_proj", "model.layers.91.mlp.experts.108.gate_proj", "model.layers.91.mlp.experts.109.gate_proj", "model.layers.91.mlp.experts.110.gate_proj", "model.layers.91.mlp.experts.111.gate_proj", "model.layers.91.mlp.experts.112.gate_proj", "model.layers.91.mlp.experts.113.gate_proj", "model.layers.91.mlp.experts.114.gate_proj", "model.layers.91.mlp.experts.115.gate_proj", "model.layers.91.mlp.experts.116.gate_proj", "model.layers.91.mlp.experts.117.gate_proj", "model.layers.91.mlp.experts.118.gate_proj", "model.layers.91.mlp.experts.119.gate_proj", "model.layers.91.mlp.experts.120.gate_proj", "model.layers.91.mlp.experts.121.gate_proj", "model.layers.91.mlp.experts.122.gate_proj", "model.layers.91.mlp.experts.123.gate_proj", "model.layers.91.mlp.experts.124.gate_proj", "model.layers.91.mlp.experts.125.gate_proj", "model.layers.91.mlp.experts.126.gate_proj", "model.layers.91.mlp.experts.127.gate_proj", "model.layers.91.mlp.experts.128.gate_proj", "model.layers.91.mlp.experts.129.gate_proj", "model.layers.91.mlp.experts.130.gate_proj", "model.layers.91.mlp.experts.131.gate_proj", "model.layers.91.mlp.experts.132.gate_proj", "model.layers.91.mlp.experts.133.gate_proj", "model.layers.91.mlp.experts.134.gate_proj", "model.layers.91.mlp.experts.135.gate_proj", "model.layers.91.mlp.experts.136.gate_proj", "model.layers.91.mlp.experts.137.gate_proj", "model.layers.91.mlp.experts.138.gate_proj", "model.layers.91.mlp.experts.139.gate_proj", "model.layers.91.mlp.experts.140.gate_proj", "model.layers.91.mlp.experts.141.gate_proj", "model.layers.91.mlp.experts.142.gate_proj", "model.layers.91.mlp.experts.143.gate_proj", "model.layers.91.mlp.experts.144.gate_proj", "model.layers.91.mlp.experts.145.gate_proj", "model.layers.91.mlp.experts.146.gate_proj", "model.layers.91.mlp.experts.147.gate_proj", "model.layers.91.mlp.experts.148.gate_proj", "model.layers.91.mlp.experts.149.gate_proj", "model.layers.91.mlp.experts.150.gate_proj", "model.layers.91.mlp.experts.151.gate_proj", "model.layers.91.mlp.experts.152.gate_proj", "model.layers.91.mlp.experts.153.gate_proj", "model.layers.91.mlp.experts.154.gate_proj", "model.layers.91.mlp.experts.155.gate_proj", "model.layers.91.mlp.experts.156.gate_proj", "model.layers.91.mlp.experts.157.gate_proj", "model.layers.91.mlp.experts.158.gate_proj", "model.layers.91.mlp.experts.159.gate_proj", "model.layers.91.mlp.experts.0.up_proj", "model.layers.91.mlp.experts.1.up_proj", "model.layers.91.mlp.experts.2.up_proj", "model.layers.91.mlp.experts.3.up_proj", "model.layers.91.mlp.experts.4.up_proj", "model.layers.91.mlp.experts.5.up_proj", "model.layers.91.mlp.experts.6.up_proj", "model.layers.91.mlp.experts.7.up_proj", "model.layers.91.mlp.experts.8.up_proj", "model.layers.91.mlp.experts.9.up_proj", "model.layers.91.mlp.experts.10.up_proj", "model.layers.91.mlp.experts.11.up_proj", "model.layers.91.mlp.experts.12.up_proj", "model.layers.91.mlp.experts.13.up_proj", "model.layers.91.mlp.experts.14.up_proj", "model.layers.91.mlp.experts.15.up_proj", "model.layers.91.mlp.experts.16.up_proj", "model.layers.91.mlp.experts.17.up_proj", "model.layers.91.mlp.experts.18.up_proj", "model.layers.91.mlp.experts.19.up_proj", "model.layers.91.mlp.experts.20.up_proj", "model.layers.91.mlp.experts.21.up_proj", "model.layers.91.mlp.experts.22.up_proj", "model.layers.91.mlp.experts.23.up_proj", "model.layers.91.mlp.experts.24.up_proj", "model.layers.91.mlp.experts.25.up_proj", "model.layers.91.mlp.experts.26.up_proj", "model.layers.91.mlp.experts.27.up_proj", "model.layers.91.mlp.experts.28.up_proj", "model.layers.91.mlp.experts.29.up_proj", "model.layers.91.mlp.experts.30.up_proj", "model.layers.91.mlp.experts.31.up_proj", "model.layers.91.mlp.experts.32.up_proj", "model.layers.91.mlp.experts.33.up_proj", "model.layers.91.mlp.experts.34.up_proj", "model.layers.91.mlp.experts.35.up_proj", "model.layers.91.mlp.experts.36.up_proj", "model.layers.91.mlp.experts.37.up_proj", "model.layers.91.mlp.experts.38.up_proj", "model.layers.91.mlp.experts.39.up_proj", "model.layers.91.mlp.experts.40.up_proj", "model.layers.91.mlp.experts.41.up_proj", "model.layers.91.mlp.experts.42.up_proj", "model.layers.91.mlp.experts.43.up_proj", "model.layers.91.mlp.experts.44.up_proj", "model.layers.91.mlp.experts.45.up_proj", "model.layers.91.mlp.experts.46.up_proj", "model.layers.91.mlp.experts.47.up_proj", "model.layers.91.mlp.experts.48.up_proj", "model.layers.91.mlp.experts.49.up_proj", "model.layers.91.mlp.experts.50.up_proj", "model.layers.91.mlp.experts.51.up_proj", "model.layers.91.mlp.experts.52.up_proj", "model.layers.91.mlp.experts.53.up_proj", "model.layers.91.mlp.experts.54.up_proj", "model.layers.91.mlp.experts.55.up_proj", "model.layers.91.mlp.experts.56.up_proj", "model.layers.91.mlp.experts.57.up_proj", "model.layers.91.mlp.experts.58.up_proj", "model.layers.91.mlp.experts.59.up_proj", "model.layers.91.mlp.experts.60.up_proj", "model.layers.91.mlp.experts.61.up_proj", "model.layers.91.mlp.experts.62.up_proj", "model.layers.91.mlp.experts.63.up_proj", "model.layers.91.mlp.experts.64.up_proj", "model.layers.91.mlp.experts.65.up_proj", "model.layers.91.mlp.experts.66.up_proj", "model.layers.91.mlp.experts.67.up_proj", "model.layers.91.mlp.experts.68.up_proj", "model.layers.91.mlp.experts.69.up_proj", "model.layers.91.mlp.experts.70.up_proj", "model.layers.91.mlp.experts.71.up_proj", "model.layers.91.mlp.experts.72.up_proj", "model.layers.91.mlp.experts.73.up_proj", "model.layers.91.mlp.experts.74.up_proj", "model.layers.91.mlp.experts.75.up_proj", "model.layers.91.mlp.experts.76.up_proj", "model.layers.91.mlp.experts.77.up_proj", "model.layers.91.mlp.experts.78.up_proj", "model.layers.91.mlp.experts.79.up_proj", "model.layers.91.mlp.experts.80.up_proj", "model.layers.91.mlp.experts.81.up_proj", "model.layers.91.mlp.experts.82.up_proj", "model.layers.91.mlp.experts.83.up_proj", "model.layers.91.mlp.experts.84.up_proj", "model.layers.91.mlp.experts.85.up_proj", "model.layers.91.mlp.experts.86.up_proj", "model.layers.91.mlp.experts.87.up_proj", "model.layers.91.mlp.experts.88.up_proj", "model.layers.91.mlp.experts.89.up_proj", "model.layers.91.mlp.experts.90.up_proj", "model.layers.91.mlp.experts.91.up_proj", "model.layers.91.mlp.experts.92.up_proj", "model.layers.91.mlp.experts.93.up_proj", "model.layers.91.mlp.experts.94.up_proj", "model.layers.91.mlp.experts.95.up_proj", "model.layers.91.mlp.experts.96.up_proj", "model.layers.91.mlp.experts.97.up_proj", "model.layers.91.mlp.experts.98.up_proj", "model.layers.91.mlp.experts.99.up_proj", "model.layers.91.mlp.experts.100.up_proj", "model.layers.91.mlp.experts.101.up_proj", "model.layers.91.mlp.experts.102.up_proj", "model.layers.91.mlp.experts.103.up_proj", "model.layers.91.mlp.experts.104.up_proj", "model.layers.91.mlp.experts.105.up_proj", "model.layers.91.mlp.experts.106.up_proj", "model.layers.91.mlp.experts.107.up_proj", "model.layers.91.mlp.experts.108.up_proj", "model.layers.91.mlp.experts.109.up_proj", "model.layers.91.mlp.experts.110.up_proj", "model.layers.91.mlp.experts.111.up_proj", "model.layers.91.mlp.experts.112.up_proj", "model.layers.91.mlp.experts.113.up_proj", "model.layers.91.mlp.experts.114.up_proj", "model.layers.91.mlp.experts.115.up_proj", "model.layers.91.mlp.experts.116.up_proj", "model.layers.91.mlp.experts.117.up_proj", "model.layers.91.mlp.experts.118.up_proj", "model.layers.91.mlp.experts.119.up_proj", "model.layers.91.mlp.experts.120.up_proj", "model.layers.91.mlp.experts.121.up_proj", "model.layers.91.mlp.experts.122.up_proj", "model.layers.91.mlp.experts.123.up_proj", "model.layers.91.mlp.experts.124.up_proj", "model.layers.91.mlp.experts.125.up_proj", "model.layers.91.mlp.experts.126.up_proj", "model.layers.91.mlp.experts.127.up_proj", "model.layers.91.mlp.experts.128.up_proj", "model.layers.91.mlp.experts.129.up_proj", "model.layers.91.mlp.experts.130.up_proj", "model.layers.91.mlp.experts.131.up_proj", "model.layers.91.mlp.experts.132.up_proj", "model.layers.91.mlp.experts.133.up_proj", "model.layers.91.mlp.experts.134.up_proj", "model.layers.91.mlp.experts.135.up_proj", "model.layers.91.mlp.experts.136.up_proj", "model.layers.91.mlp.experts.137.up_proj", "model.layers.91.mlp.experts.138.up_proj", "model.layers.91.mlp.experts.139.up_proj", "model.layers.91.mlp.experts.140.up_proj", "model.layers.91.mlp.experts.141.up_proj", "model.layers.91.mlp.experts.142.up_proj", "model.layers.91.mlp.experts.143.up_proj", "model.layers.91.mlp.experts.144.up_proj", "model.layers.91.mlp.experts.145.up_proj", "model.layers.91.mlp.experts.146.up_proj", "model.layers.91.mlp.experts.147.up_proj", "model.layers.91.mlp.experts.148.up_proj", "model.layers.91.mlp.experts.149.up_proj", "model.layers.91.mlp.experts.150.up_proj", "model.layers.91.mlp.experts.151.up_proj", "model.layers.91.mlp.experts.152.up_proj", "model.layers.91.mlp.experts.153.up_proj", "model.layers.91.mlp.experts.154.up_proj", "model.layers.91.mlp.experts.155.up_proj", "model.layers.91.mlp.experts.156.up_proj", "model.layers.91.mlp.experts.157.up_proj", "model.layers.91.mlp.experts.158.up_proj", "model.layers.91.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.793518645689108e-07, "dbits": 5033164800 } ] }, { "idx": 548, "layers": [ "model.layers.91.mlp.experts.0.down_proj", "model.layers.91.mlp.experts.1.down_proj", "model.layers.91.mlp.experts.2.down_proj", "model.layers.91.mlp.experts.3.down_proj", "model.layers.91.mlp.experts.4.down_proj", "model.layers.91.mlp.experts.5.down_proj", "model.layers.91.mlp.experts.6.down_proj", "model.layers.91.mlp.experts.7.down_proj", "model.layers.91.mlp.experts.8.down_proj", "model.layers.91.mlp.experts.9.down_proj", "model.layers.91.mlp.experts.10.down_proj", "model.layers.91.mlp.experts.11.down_proj", "model.layers.91.mlp.experts.12.down_proj", "model.layers.91.mlp.experts.13.down_proj", "model.layers.91.mlp.experts.14.down_proj", "model.layers.91.mlp.experts.15.down_proj", "model.layers.91.mlp.experts.16.down_proj", "model.layers.91.mlp.experts.17.down_proj", "model.layers.91.mlp.experts.18.down_proj", "model.layers.91.mlp.experts.19.down_proj", "model.layers.91.mlp.experts.20.down_proj", "model.layers.91.mlp.experts.21.down_proj", "model.layers.91.mlp.experts.22.down_proj", "model.layers.91.mlp.experts.23.down_proj", "model.layers.91.mlp.experts.24.down_proj", "model.layers.91.mlp.experts.25.down_proj", "model.layers.91.mlp.experts.26.down_proj", "model.layers.91.mlp.experts.27.down_proj", "model.layers.91.mlp.experts.28.down_proj", "model.layers.91.mlp.experts.29.down_proj", "model.layers.91.mlp.experts.30.down_proj", "model.layers.91.mlp.experts.31.down_proj", "model.layers.91.mlp.experts.32.down_proj", "model.layers.91.mlp.experts.33.down_proj", "model.layers.91.mlp.experts.34.down_proj", "model.layers.91.mlp.experts.35.down_proj", "model.layers.91.mlp.experts.36.down_proj", "model.layers.91.mlp.experts.37.down_proj", "model.layers.91.mlp.experts.38.down_proj", "model.layers.91.mlp.experts.39.down_proj", "model.layers.91.mlp.experts.40.down_proj", "model.layers.91.mlp.experts.41.down_proj", "model.layers.91.mlp.experts.42.down_proj", "model.layers.91.mlp.experts.43.down_proj", "model.layers.91.mlp.experts.44.down_proj", "model.layers.91.mlp.experts.45.down_proj", "model.layers.91.mlp.experts.46.down_proj", "model.layers.91.mlp.experts.47.down_proj", "model.layers.91.mlp.experts.48.down_proj", "model.layers.91.mlp.experts.49.down_proj", "model.layers.91.mlp.experts.50.down_proj", "model.layers.91.mlp.experts.51.down_proj", "model.layers.91.mlp.experts.52.down_proj", "model.layers.91.mlp.experts.53.down_proj", "model.layers.91.mlp.experts.54.down_proj", "model.layers.91.mlp.experts.55.down_proj", "model.layers.91.mlp.experts.56.down_proj", "model.layers.91.mlp.experts.57.down_proj", "model.layers.91.mlp.experts.58.down_proj", "model.layers.91.mlp.experts.59.down_proj", "model.layers.91.mlp.experts.60.down_proj", "model.layers.91.mlp.experts.61.down_proj", "model.layers.91.mlp.experts.62.down_proj", "model.layers.91.mlp.experts.63.down_proj", "model.layers.91.mlp.experts.64.down_proj", "model.layers.91.mlp.experts.65.down_proj", "model.layers.91.mlp.experts.66.down_proj", "model.layers.91.mlp.experts.67.down_proj", "model.layers.91.mlp.experts.68.down_proj", "model.layers.91.mlp.experts.69.down_proj", "model.layers.91.mlp.experts.70.down_proj", "model.layers.91.mlp.experts.71.down_proj", "model.layers.91.mlp.experts.72.down_proj", "model.layers.91.mlp.experts.73.down_proj", "model.layers.91.mlp.experts.74.down_proj", "model.layers.91.mlp.experts.75.down_proj", "model.layers.91.mlp.experts.76.down_proj", "model.layers.91.mlp.experts.77.down_proj", "model.layers.91.mlp.experts.78.down_proj", "model.layers.91.mlp.experts.79.down_proj", "model.layers.91.mlp.experts.80.down_proj", "model.layers.91.mlp.experts.81.down_proj", "model.layers.91.mlp.experts.82.down_proj", "model.layers.91.mlp.experts.83.down_proj", "model.layers.91.mlp.experts.84.down_proj", "model.layers.91.mlp.experts.85.down_proj", "model.layers.91.mlp.experts.86.down_proj", "model.layers.91.mlp.experts.87.down_proj", "model.layers.91.mlp.experts.88.down_proj", "model.layers.91.mlp.experts.89.down_proj", "model.layers.91.mlp.experts.90.down_proj", "model.layers.91.mlp.experts.91.down_proj", "model.layers.91.mlp.experts.92.down_proj", "model.layers.91.mlp.experts.93.down_proj", "model.layers.91.mlp.experts.94.down_proj", "model.layers.91.mlp.experts.95.down_proj", "model.layers.91.mlp.experts.96.down_proj", "model.layers.91.mlp.experts.97.down_proj", "model.layers.91.mlp.experts.98.down_proj", "model.layers.91.mlp.experts.99.down_proj", "model.layers.91.mlp.experts.100.down_proj", "model.layers.91.mlp.experts.101.down_proj", "model.layers.91.mlp.experts.102.down_proj", "model.layers.91.mlp.experts.103.down_proj", "model.layers.91.mlp.experts.104.down_proj", "model.layers.91.mlp.experts.105.down_proj", "model.layers.91.mlp.experts.106.down_proj", "model.layers.91.mlp.experts.107.down_proj", "model.layers.91.mlp.experts.108.down_proj", "model.layers.91.mlp.experts.109.down_proj", "model.layers.91.mlp.experts.110.down_proj", "model.layers.91.mlp.experts.111.down_proj", "model.layers.91.mlp.experts.112.down_proj", "model.layers.91.mlp.experts.113.down_proj", "model.layers.91.mlp.experts.114.down_proj", "model.layers.91.mlp.experts.115.down_proj", "model.layers.91.mlp.experts.116.down_proj", "model.layers.91.mlp.experts.117.down_proj", "model.layers.91.mlp.experts.118.down_proj", "model.layers.91.mlp.experts.119.down_proj", "model.layers.91.mlp.experts.120.down_proj", "model.layers.91.mlp.experts.121.down_proj", "model.layers.91.mlp.experts.122.down_proj", "model.layers.91.mlp.experts.123.down_proj", "model.layers.91.mlp.experts.124.down_proj", "model.layers.91.mlp.experts.125.down_proj", "model.layers.91.mlp.experts.126.down_proj", "model.layers.91.mlp.experts.127.down_proj", "model.layers.91.mlp.experts.128.down_proj", "model.layers.91.mlp.experts.129.down_proj", "model.layers.91.mlp.experts.130.down_proj", "model.layers.91.mlp.experts.131.down_proj", "model.layers.91.mlp.experts.132.down_proj", "model.layers.91.mlp.experts.133.down_proj", "model.layers.91.mlp.experts.134.down_proj", "model.layers.91.mlp.experts.135.down_proj", "model.layers.91.mlp.experts.136.down_proj", "model.layers.91.mlp.experts.137.down_proj", "model.layers.91.mlp.experts.138.down_proj", "model.layers.91.mlp.experts.139.down_proj", "model.layers.91.mlp.experts.140.down_proj", "model.layers.91.mlp.experts.141.down_proj", "model.layers.91.mlp.experts.142.down_proj", "model.layers.91.mlp.experts.143.down_proj", "model.layers.91.mlp.experts.144.down_proj", "model.layers.91.mlp.experts.145.down_proj", "model.layers.91.mlp.experts.146.down_proj", "model.layers.91.mlp.experts.147.down_proj", "model.layers.91.mlp.experts.148.down_proj", "model.layers.91.mlp.experts.149.down_proj", "model.layers.91.mlp.experts.150.down_proj", "model.layers.91.mlp.experts.151.down_proj", "model.layers.91.mlp.experts.152.down_proj", "model.layers.91.mlp.experts.153.down_proj", "model.layers.91.mlp.experts.154.down_proj", "model.layers.91.mlp.experts.155.down_proj", "model.layers.91.mlp.experts.156.down_proj", "model.layers.91.mlp.experts.157.down_proj", "model.layers.91.mlp.experts.158.down_proj", "model.layers.91.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 2.445923746563516e-06, "dbits": 2516582400 } ] } ], "base_kld": 0.0009723345719976351, "arch_string": "Glm4MoeForCausalLM" }