diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json"
new file mode 100644--- /dev/null
+++ "b/reduced_ppl_average.json"
@@ -0,0 +1,13520 @@
+{
+    "original_ppl": 7.836853981018066,
+    "reduced_ppl": [
+        {
+            "bit": 1,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 100028.296875
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 83601.1640625
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 92061.140625
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 73126.3125
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 88826.78125
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 88911.1953125
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 101415.1328125
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 112060.359375
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 99527.3984375
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 112942.484375
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 45329.7890625
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 103545.53125
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 75738.6953125
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 129706.734375
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 92175.9609375
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 93578.515625
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 124439.2734375
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 112074.7890625
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 75031.5625
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 62521.05859375
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 74879.6640625
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 104447.65625
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 87225.9296875
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 86935.265625
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 94477.2890625
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 137997.0625
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 85168.9453125
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 106327.9609375
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 111321.1328125
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 93278.4296875
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 109165.90625
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 60157.2265625
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 112575.2578125
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 63213.53125
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 93929.5390625
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 114455.203125
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 102396.4921875
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 114803.4921875
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 106479.578125
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 99373.9453125
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 104597.9765625
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 132235.328125
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 98028.671875
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 123878.734375
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 52708.54296875
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 160839.015625
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 119890.265625
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 142591.71875
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 54418.26953125
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 110809.1171875
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 112228.265625
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 144154.25
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 139704.3125
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 138363.53125
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 104079.5546875
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 73313.3125
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 96951.3046875
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 99314.6328125
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 86732.0546875
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 69419.7265625
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 78901.5546875
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 100445.65625
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 96061.5078125
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 90961.0859375
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 91467.8046875
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 142583.0
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 129425.375
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 150609.171875
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 76623.3671875
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 109846.7109375
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 102881.8046875
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 114216.078125
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 102605.484375
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 84844.3515625
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 132192.453125
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 87694.3515625
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 88012.4765625
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 88757.1796875
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 83935.09375
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 82636.4765625
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 131119.703125
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 135234.796875
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 85947.2734375
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 139499.9375
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 120826.1953125
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 103897.96875
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 138357.59375
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 108792.1796875
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 110489.28125
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 67052.59375
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 112562.2734375
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 110496.234375
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 76167.9453125
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 140275.96875
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 92528.5234375
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 121607.4296875
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 70721.2265625
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 80244.796875
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 93308.234375
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 96368.90625
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 78115.1796875
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 75864.1953125
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 94399.03125
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 123373.421875
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 112834.1875
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 98729.703125
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 89207.8671875
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 116006.96875
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 127953.7578125
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 90116.734375
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 95834.578125
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 77265.65625
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 99592.3515625
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 96327.921875
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 104385.4140625
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 111820.796875
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 117226.765625
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 66718.859375
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 92781.234375
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 95134.5078125
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 91213.78125
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 130175.515625
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 94601.625
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 95593.421875
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 92860.4609375
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 100756.3203125
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 102315.859375
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 100177.3203125
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 94189.234375
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 92713.1328125
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 119000.3828125
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 94653.5078125
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 95588.8671875
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 100924.609375
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 102325.421875
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 110875.71875
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 89535.2421875
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 104271.0
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 109607.703125
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 116153.875
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 103546.421875
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 103352.0625
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 104164.4609375
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 104534.25
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 116952.1796875
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 81080.890625
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 111908.0625
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 105461.484375
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 104852.2578125
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 100538.625
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 92973.625
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 99675.390625
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 96183.25
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 100381.8828125
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 105332.9296875
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 107246.1484375
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 111774.2109375
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 107385.75
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 101985.6171875
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 98287.2265625
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 101858.2890625
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 102961.8984375
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 104293.0703125
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 108741.25
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 97056.859375
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 105336.4453125
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 93973.5390625
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 95834.03125
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 103708.1015625
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 106327.15625
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 92967.6796875
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 117210.109375
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 121469.3828125
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 98738.3671875
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 103587.109375
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 102603.234375
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 104255.4921875
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 122396.390625
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 107528.8125
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 105061.75
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 102482.4609375
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 105272.9765625
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 105902.125
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 106416.4296875
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 104918.875
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 111305.640625
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 102500.8359375
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 100910.359375
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 99575.4375
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 100629.3671875
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 99075.578125
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 106700.5625
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 92074.921875
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 102103.1796875
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 97920.3828125
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 107986.6328125
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 103662.4140625
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 105780.1875
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 102874.9375
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 103913.4296875
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 101132.2421875
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 101583.75
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 102753.15625
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 102017.5234375
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 103262.015625
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 103129.75
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 97783.9453125
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 106424.953125
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 105678.7578125
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 98854.640625
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 103172.4453125
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 102745.4140625
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 107614.578125
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 109414.8046875
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 88412.921875
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 93250.3203125
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 103805.0703125
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 107196.25
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 104741.3203125
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 102185.1015625
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 117909.1328125
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 104212.1484375
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 119181.8828125
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 116339.125
+                }
+            ]
+        },
+        {
+            "bit": 2,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 228.3678741455078
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 228.46743774414062
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 222.2572021484375
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 211.26962280273438
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 215.52572631835938
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 219.65652465820312
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 209.06570434570312
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 223.21820068359375
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 221.11788940429688
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 213.39540100097656
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 209.99407958984375
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 218.8164520263672
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 219.75196838378906
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 221.50559997558594
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 188.50621032714844
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 187.1911163330078
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 233.6358184814453
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 226.49337768554688
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 214.4426727294922
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 214.77731323242188
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 215.86720275878906
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 217.47842407226562
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 225.78201293945312
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 221.66905212402344
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 217.09828186035156
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 215.488525390625
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 227.7002410888672
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 211.12149047851562
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 222.66986083984375
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 221.27883911132812
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 235.73121643066406
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 219.1390838623047
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 209.91036987304688
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 208.96144104003906
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 223.18743896484375
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 219.43960571289062
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 226.047119140625
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 222.2498016357422
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 222.46185302734375
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 215.72518920898438
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 208.7921142578125
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 212.58201599121094
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 222.00534057617188
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 224.95497131347656
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 216.35894775390625
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 218.5930633544922
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 216.84913635253906
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 221.52291870117188
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 216.25375366210938
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 217.30572509765625
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 223.11370849609375
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 216.8362274169922
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 226.0778350830078
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 221.59730529785156
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 214.74014282226562
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 222.6441650390625
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 222.92185974121094
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 223.34532165527344
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 223.4870147705078
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 219.4259033203125
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 219.3214111328125
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 229.59971618652344
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 221.233154296875
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 227.65899658203125
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 224.48294067382812
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 223.63145446777344
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 213.0618133544922
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 221.92742919921875
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 221.87855529785156
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 218.923095703125
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 229.1191864013672
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 225.0966033935547
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 227.00900268554688
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 216.30572509765625
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 221.0341796875
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 220.5524139404297
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 226.80699157714844
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 217.80690002441406
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 224.61444091796875
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 222.60679626464844
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 216.32872009277344
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 218.53457641601562
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 218.1864013671875
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 222.09829711914062
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 225.72451782226562
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 225.82896423339844
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 218.0876922607422
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 209.11355590820312
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 221.56707763671875
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 213.20074462890625
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 219.24945068359375
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 223.7906036376953
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 223.98544311523438
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 219.54417419433594
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 219.72315979003906
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 218.4221649169922
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 225.51979064941406
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 230.84043884277344
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 217.7310791015625
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 220.02069091796875
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 220.4967803955078
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 218.07510375976562
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 217.10997009277344
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 216.0346221923828
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 222.48519897460938
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 221.45236206054688
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 221.23158264160156
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 231.26304626464844
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 224.46058654785156
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 224.06471252441406
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 224.08885192871094
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 222.9420623779297
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 223.28207397460938
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 225.77210998535156
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 218.97447204589844
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 220.190185546875
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 221.8339080810547
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 223.5560760498047
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 225.9592742919922
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 223.25970458984375
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 225.0081787109375
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 225.51344299316406
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 226.3449249267578
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 223.1551971435547
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 222.52784729003906
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 221.00045776367188
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 225.70602416992188
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 225.899169921875
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 225.85479736328125
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 226.9906005859375
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 226.323974609375
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 224.31121826171875
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 227.20437622070312
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 226.37547302246094
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 225.5098876953125
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 228.6000518798828
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 226.0824737548828
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 226.59654235839844
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 225.6342315673828
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 223.24417114257812
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 225.87008666992188
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 225.52528381347656
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 224.75511169433594
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 226.6101531982422
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 223.4763641357422
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 227.87294006347656
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 225.22415161132812
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 226.0972442626953
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 225.25572204589844
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 226.89451599121094
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 224.33132934570312
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 226.04518127441406
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 226.2178192138672
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 226.20896911621094
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 226.49888610839844
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 226.83848571777344
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 227.25509643554688
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 227.216064453125
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 227.22634887695312
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 226.32937622070312
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 225.7090301513672
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 225.5762481689453
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 226.71055603027344
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 225.86599731445312
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 225.4437713623047
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 226.06124877929688
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 225.97427368164062
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 225.8191680908203
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 225.64617919921875
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 225.4522705078125
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 226.14308166503906
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 227.13287353515625
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 225.8455352783203
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 225.1112060546875
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 225.62767028808594
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 225.1337432861328
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 224.3606414794922
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 224.99853515625
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 224.58328247070312
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 225.21051025390625
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 225.2963409423828
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 225.1029510498047
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 227.73324584960938
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 227.08338928222656
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 226.2667999267578
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 225.4050750732422
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 225.28195190429688
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 225.0098876953125
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 225.7315216064453
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 223.64349365234375
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 225.79006958007812
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 227.4889373779297
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 227.6394500732422
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 228.1409454345703
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 229.07460021972656
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 226.2749786376953
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 225.8909912109375
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 226.37998962402344
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 226.64256286621094
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 225.7344207763672
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 225.33372497558594
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 225.9529266357422
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 227.65052795410156
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 228.2270050048828
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 227.05838012695312
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 226.4374237060547
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 224.9395294189453
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 226.60313415527344
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 229.72567749023438
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 226.14036560058594
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 225.7740478515625
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 226.18544006347656
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 227.56976318359375
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 227.54080200195312
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 223.43821716308594
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 226.6251678466797
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 224.72274780273438
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 224.64474487304688
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 226.2241668701172
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 225.1304168701172
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 226.35560607910156
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 229.4119415283203
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 233.0724334716797
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 222.42938232421875
+                }
+            ]
+        },
+        {
+            "bit": 3,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 13.056154251098633
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 13.054343223571777
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 13.077469825744629
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 13.02145004272461
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 13.039437294006348
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 13.01038932800293
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 12.998372077941895
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 13.039815902709961
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 13.037182807922363
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 13.07637882232666
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 13.04530143737793
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 13.008739471435547
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 13.04500961303711
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 13.011906623840332
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 12.994876861572266
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 12.986613273620605
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 13.01194953918457
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 13.030412673950195
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 13.009800910949707
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 13.036300659179688
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 12.98751163482666
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 13.033370018005371
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 13.044224739074707
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 13.043557167053223
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 13.003244400024414
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 13.011345863342285
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 13.079253196716309
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 13.028091430664062
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 13.058588981628418
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 13.051467895507812
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 13.022072792053223
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 13.00686264038086
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 12.993618965148926
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 13.017637252807617
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 13.031915664672852
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 13.042054176330566
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 13.038243293762207
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 13.079460144042969
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 13.074601173400879
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 13.008724212646484
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 13.00227165222168
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 13.04793643951416
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 13.020977973937988
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 13.046019554138184
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 13.048234939575195
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 13.06552791595459
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 12.98932933807373
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 13.05797290802002
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 13.01521110534668
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 13.025771141052246
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 13.060395240783691
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 13.020121574401855
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 13.044026374816895
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 13.012030601501465
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 12.986647605895996
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 13.008950233459473
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 13.072696685791016
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 13.080307960510254
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 13.080721855163574
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 13.017451286315918
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 13.000194549560547
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 13.049578666687012
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 13.001651763916016
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 13.059977531433105
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 13.026103973388672
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 13.0299711227417
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 13.05820369720459
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 13.023719787597656
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 13.056319236755371
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 13.070926666259766
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 13.03841781616211
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 13.06108570098877
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 13.035017013549805
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 13.08551025390625
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 13.042689323425293
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 13.03673267364502
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 13.024617195129395
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 13.04966926574707
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 13.044231414794922
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 13.021648406982422
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 13.010563850402832
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 13.015127182006836
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 13.090855598449707
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 13.024585723876953
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 13.044548988342285
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 13.041081428527832
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 13.012505531311035
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 13.02734661102295
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 12.987951278686523
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 13.04334831237793
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 13.009236335754395
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 13.046123504638672
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 13.039216041564941
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 13.055307388305664
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 12.991779327392578
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 13.042813301086426
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 13.034221649169922
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 13.03722095489502
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 13.006742477416992
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 12.986577033996582
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 12.999540328979492
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 13.008174896240234
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 13.015645980834961
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 12.988312721252441
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 12.993650436401367
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 13.044897079467773
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 13.054044723510742
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 13.078455924987793
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 13.01488208770752
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 13.013466835021973
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 13.058675765991211
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 13.0155029296875
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 13.04500675201416
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 13.058393478393555
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 13.05136775970459
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 13.07390022277832
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 12.98832893371582
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 13.051324844360352
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 13.004103660583496
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 13.049027442932129
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 13.056253433227539
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 13.066168785095215
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 13.041265487670898
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 13.032549858093262
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 13.050658226013184
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 13.03137493133545
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 13.055743217468262
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 13.054336547851562
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 13.049675941467285
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 13.060420036315918
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 13.029616355895996
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 13.062712669372559
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 13.030570983886719
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 13.064163208007812
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 13.05705738067627
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 13.062189102172852
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 13.059563636779785
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 13.034463882446289
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 13.056951522827148
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 13.029101371765137
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 13.060152053833008
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 13.057300567626953
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 13.051545143127441
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 13.062098503112793
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 13.025392532348633
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 13.053240776062012
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 13.028526306152344
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 13.063642501831055
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 13.04793930053711
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 13.047802925109863
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 13.056182861328125
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 13.037148475646973
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 13.048698425292969
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 13.021349906921387
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 13.05784797668457
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 13.05808162689209
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 13.057001113891602
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 13.054491996765137
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 13.028918266296387
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 13.045639991760254
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 13.028182029724121
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 13.055662155151367
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 13.06143569946289
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 13.05565071105957
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 13.052151679992676
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 13.033097267150879
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 13.045221328735352
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 13.033075332641602
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 13.057244300842285
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 13.064075469970703
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 13.062954902648926
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 13.067680358886719
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 13.041974067687988
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 13.059398651123047
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 13.035305976867676
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 13.070158004760742
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 13.057792663574219
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 13.052410125732422
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 13.053427696228027
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 13.045764923095703
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 13.04421329498291
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 13.043901443481445
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 13.06135082244873
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 13.059280395507812
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 13.084874153137207
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 13.055121421813965
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 13.039306640625
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 13.042468070983887
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 13.052369117736816
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 13.040509223937988
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 13.05801010131836
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 13.061326026916504
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 13.035066604614258
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 13.062952041625977
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 13.074411392211914
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 13.057586669921875
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 13.049267768859863
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 13.039971351623535
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 13.053409576416016
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 13.047802925109863
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 13.037298202514648
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 13.0574312210083
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 13.04185962677002
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 13.041374206542969
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 13.051180839538574
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 13.051349639892578
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 13.047565460205078
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 13.035026550292969
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 13.035231590270996
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 13.044760704040527
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 13.061020851135254
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 13.067069053649902
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 13.070731163024902
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 13.072245597839355
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 13.043367385864258
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 13.033942222595215
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 13.034234046936035
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 13.090128898620605
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 12.994473457336426
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 13.06491470336914
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 13.013504028320312
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 13.11275863647461
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 13.060267448425293
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 13.00472354888916
+                }
+            ]
+        },
+        {
+            "bit": 4,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 9.58368968963623
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 9.58467960357666
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 9.580440521240234
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 9.57874584197998
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 9.574732780456543
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 9.582172393798828
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 9.575645446777344
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 9.584466934204102
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 9.579532623291016
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 9.580706596374512
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 9.576375961303711
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 9.57551383972168
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 9.585015296936035
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 9.580575942993164
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 9.574075698852539
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 9.571317672729492
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 9.582648277282715
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 9.583954811096191
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 9.578856468200684
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 9.57884407043457
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 9.582026481628418
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 9.5790433883667
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 9.57956600189209
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 9.583588600158691
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 9.580179214477539
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 9.576765060424805
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 9.578166961669922
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 9.576703071594238
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 9.578557014465332
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 9.586102485656738
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 9.578678131103516
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 9.58650016784668
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 9.571772575378418
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 9.566763877868652
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 9.570877075195312
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 9.58657169342041
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 9.5878267288208
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 9.595057487487793
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 9.573486328125
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 9.576189041137695
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 9.584802627563477
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 9.575309753417969
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 9.583090782165527
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 9.584096908569336
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 9.57120132446289
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 9.573803901672363
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 9.575766563415527
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 9.580605506896973
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 9.575380325317383
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 9.585211753845215
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 9.584115028381348
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 9.576613426208496
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 9.577385902404785
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 9.568405151367188
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 9.573129653930664
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 9.57342529296875
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 9.588640213012695
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 9.584197044372559
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 9.581779479980469
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 9.594022750854492
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 9.579865455627441
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 9.578737258911133
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 9.58065414428711
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 9.585723876953125
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 9.581640243530273
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 9.582767486572266
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 9.578643798828125
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 9.57645320892334
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 9.581679344177246
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 9.571950912475586
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 9.582230567932129
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 9.585735321044922
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 9.580893516540527
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 9.578568458557129
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 9.576929092407227
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 9.585309982299805
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 9.583157539367676
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 9.589150428771973
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 9.579998970031738
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 9.588098526000977
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 9.57970142364502
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 9.584339141845703
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 9.577471733093262
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 9.5828218460083
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 9.580427169799805
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 9.581343650817871
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 9.592024803161621
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 9.565947532653809
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 9.574755668640137
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 9.58171558380127
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 9.580986976623535
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 9.58011245727539
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 9.580998420715332
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 9.581738471984863
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 9.575011253356934
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 9.574139595031738
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 9.589289665222168
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 9.571155548095703
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 9.569360733032227
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 9.566009521484375
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 9.57751750946045
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 9.577122688293457
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 9.576403617858887
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 9.582718849182129
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 9.576411247253418
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 9.584362030029297
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 9.569990158081055
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 9.585709571838379
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 9.577430725097656
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 9.578965187072754
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 9.581873893737793
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 9.577299118041992
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 9.583015441894531
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 9.582571983337402
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 9.581756591796875
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 9.582767486572266
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 9.5734224319458
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 9.580879211425781
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 9.573946952819824
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 9.585426330566406
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 9.584023475646973
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 9.576467514038086
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 9.587026596069336
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 9.577569961547852
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 9.58086109161377
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 9.577326774597168
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 9.584494590759277
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 9.585563659667969
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 9.580806732177734
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 9.581672668457031
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 9.578673362731934
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 9.581670761108398
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 9.57507038116455
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 9.584589958190918
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 9.580304145812988
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 9.582133293151855
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 9.586957931518555
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 9.577367782592773
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 9.580398559570312
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 9.578797340393066
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 9.58255386352539
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 9.585431098937988
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 9.585015296936035
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 9.583252906799316
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 9.57909870147705
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 9.582938194274902
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 9.57776927947998
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 9.582714080810547
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 9.580775260925293
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 9.582746505737305
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 9.581720352172852
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 9.579166412353516
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 9.581975936889648
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 9.577317237854004
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 9.585061073303223
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 9.583849906921387
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 9.582679748535156
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 9.582245826721191
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 9.576380729675293
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 9.584588050842285
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 9.5770845413208
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 9.583905220031738
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 9.585531234741211
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 9.584128379821777
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 9.5820951461792
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 9.577651977539062
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 9.580744743347168
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 9.579171180725098
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 9.58439826965332
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 9.58620834350586
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 9.583828926086426
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 9.584311485290527
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 9.57903003692627
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 9.5830659866333
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 9.579668998718262
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 9.585648536682129
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 9.581844329833984
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 9.583584785461426
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 9.582191467285156
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 9.578065872192383
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 9.577641487121582
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 9.577990531921387
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 9.584452629089355
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 9.584325790405273
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 9.586766242980957
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 9.586427688598633
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 9.578373908996582
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 9.576102256774902
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 9.5807523727417
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 9.580916404724121
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 9.58535385131836
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 9.58250617980957
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 9.586270332336426
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 9.582684516906738
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 9.584512710571289
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 9.578536987304688
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 9.583807945251465
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 9.582019805908203
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 9.583738327026367
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 9.581379890441895
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 9.577508926391602
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 9.580514907836914
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 9.584270477294922
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 9.57970142364502
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 9.586164474487305
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 9.586624145507812
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 9.58095932006836
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 9.582301139831543
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 9.585453987121582
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 9.58311653137207
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 9.585872650146484
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 9.585087776184082
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 9.58742904663086
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 9.585756301879883
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 9.58178424835205
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 9.58070182800293
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 9.578216552734375
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 9.58604621887207
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 9.582011222839355
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 9.58640193939209
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 9.582961082458496
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 9.583719253540039
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 9.59237003326416
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 9.559793472290039
+                }
+            ]
+        },
+        {
+            "bit": 5,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.778406143188477
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.777447700500488
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.779881477355957
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.776079177856445
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.778031349182129
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.775751113891602
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.775178909301758
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.77658748626709
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.777767181396484
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.777591705322266
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.777206420898438
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.774002075195312
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.777462005615234
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.776689529418945
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.776951789855957
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.776296615600586
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.778401374816895
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.779263496398926
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.77534008026123
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.778318405151367
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.774453163146973
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.777474403381348
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.777976989746094
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.775001525878906
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.780559539794922
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.774344444274902
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.7772216796875
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.776796340942383
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.778830528259277
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.77869701385498
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.777999877929688
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.777775764465332
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.772913932800293
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.77275276184082
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.777854919433594
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.778979301452637
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.774384498596191
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.775641441345215
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.781765937805176
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.775938034057617
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.780227661132812
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.774049758911133
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.780594825744629
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.777068138122559
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.779412269592285
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.78101634979248
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.776592254638672
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.774290084838867
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.775653839111328
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.777694702148438
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.778154373168945
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.785224914550781
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.7829008102417
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.77574634552002
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.774269104003906
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.772528648376465
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.777868270874023
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.77815055847168
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.778684616088867
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.779923439025879
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.776885986328125
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.777661323547363
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.780928611755371
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.778655052185059
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.778273582458496
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.77935791015625
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.777143478393555
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.778282165527344
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.776110649108887
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.7742338180542
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.777093887329102
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.779685020446777
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.778273582458496
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.77972412109375
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.777543067932129
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.779157638549805
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.777327537536621
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.777491569519043
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.778324127197266
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.777381896972656
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.778169631958008
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.77391529083252
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.78087329864502
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.776725769042969
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.775616645812988
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.777050018310547
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.775091171264648
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.778483390808105
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.776082992553711
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.778600692749023
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.776304244995117
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.778600692749023
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.774917602539062
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.776198387145996
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.7755765914917
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.775080680847168
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.778006553649902
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.776344299316406
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.77680492401123
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.76762580871582
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.775055885314941
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.775625228881836
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.776091575622559
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.778504371643066
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.77372932434082
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.778454780578613
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.768485069274902
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.773797035217285
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.778748512268066
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.776223182678223
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.778509140014648
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.77810001373291
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.777725219726562
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.778265953063965
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.778481483459473
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.781338691711426
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.774619102478027
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.778136253356934
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.776625633239746
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.778306007385254
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.778523445129395
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.775733947753906
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.778860092163086
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.774869918823242
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.776144027709961
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.777863502502441
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.778523445129395
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.777863502502441
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.778639793395996
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.77767562866211
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.776018142700195
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.779090881347656
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.777615547180176
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.779667854309082
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.775691986083984
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.778306007385254
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.780230522155762
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.776361465454102
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.77888298034668
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.774127006530762
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.779196739196777
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.778435707092285
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.778849601745605
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.778772354125977
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.775938034057617
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.778173446655273
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.776731491088867
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.777587890625
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.779818534851074
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.77871322631836
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.779890060424805
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.776871681213379
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.775895118713379
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.776741981506348
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.777691841125488
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.779154777526855
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.778847694396973
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.778868675231934
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.776177406311035
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.7761812210083
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.776966094970703
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.77811050415039
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.77846908569336
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.778206825256348
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.7781400680542
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.775520324707031
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.7780179977417
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.776382446289062
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.778709411621094
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.778554916381836
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.779345512390137
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.778998374938965
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.776545524597168
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.776223182678223
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.777604103088379
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.778119087219238
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.777512550354004
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.779128074645996
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.778240203857422
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.776174545288086
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.777068138122559
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.77651596069336
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.778345108032227
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.778504371643066
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.779935836791992
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.779147148132324
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.777303695678711
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.777006149291992
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.776315689086914
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.7772798538208
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.777524948120117
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.779046058654785
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.777365684509277
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.777355194091797
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.779129981994629
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.777484893798828
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.777809143066406
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.779004096984863
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.778230667114258
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.778755187988281
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.77780532836914
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.778571128845215
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.777223587036133
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.781336784362793
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.779497146606445
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.780034065246582
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.777717590332031
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.778623580932617
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.775245666503906
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.777462005615234
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.778960227966309
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.77929973602295
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.78048038482666
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.7788724899292
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.777252197265625
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.776972770690918
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.775493621826172
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.779329299926758
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.776165008544922
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.77793025970459
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.775813102722168
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.776302337646484
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.767470359802246
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.766407012939453
+                }
+            ]
+        },
+        {
+            "bit": 6,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.446268081665039
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.445901870727539
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.444463729858398
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.44515323638916
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.445365905761719
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.446763038635254
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.445833206176758
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.446557998657227
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.446335792541504
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.445557594299316
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.445961952209473
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.445184707641602
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.444459915161133
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.446352005004883
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.44485855102539
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.44564437866211
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.443868637084961
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.443276405334473
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.445685386657715
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.443832397460938
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.445220947265625
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.446005821228027
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.446074485778809
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.445877075195312
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.445510864257812
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.4454345703125
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.445760726928711
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.445148468017578
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.44466495513916
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.44526481628418
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.444544792175293
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.445422172546387
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.445068359375
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.446451187133789
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.445416450500488
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.445396423339844
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.449325561523438
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.449925422668457
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.446422576904297
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.445442199707031
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.445683479309082
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.446860313415527
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.446666717529297
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.446847915649414
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.44710636138916
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.443880081176758
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.443854331970215
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.445442199707031
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.44526195526123
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.44786262512207
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.447061538696289
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.444379806518555
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.444355010986328
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.443209648132324
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.442265510559082
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.44590950012207
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.447347640991211
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.447166442871094
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.44661808013916
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.44727897644043
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.445452690124512
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.445460319519043
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.44485092163086
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.44685173034668
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.445730209350586
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.446557998657227
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.448132514953613
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.444609642028809
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.447771072387695
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.446396827697754
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.44629955291748
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.446183204650879
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.447311401367188
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.445523262023926
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.445284843444824
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.445409774780273
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.44580078125
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.446779251098633
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.446304321289062
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.447322845458984
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.44529914855957
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.447250366210938
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.446781158447266
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.445295333862305
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.44620132446289
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.445683479309082
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.443924903869629
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.44295883178711
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.446123123168945
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.445488929748535
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.446090698242188
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.445220947265625
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.445404052734375
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.44498348236084
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.444526672363281
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.44593334197998
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.446650505065918
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.444852828979492
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.444454193115234
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.442167282104492
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.445760726928711
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.444586753845215
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.444647789001465
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.445494651794434
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.4448823928833
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.446949005126953
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.441555976867676
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.447705268859863
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.445769309997559
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.44493293762207
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.445792198181152
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.442906379699707
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.446367263793945
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.446304321289062
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.445438385009766
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.444693565368652
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.445124626159668
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.44657039642334
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.444841384887695
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.446453094482422
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.447112083435059
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.446263313293457
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.445061683654785
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.445284843444824
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.446332931518555
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.444198608398438
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.44651985168457
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.446598052978516
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.446005821228027
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.445953369140625
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.445769309997559
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.445100784301758
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.444735527038574
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.446372985839844
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.445813179016113
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.445708274841309
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.446630477905273
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.444903373718262
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.445507049560547
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.446324348449707
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.446215629577637
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.447033882141113
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.44589900970459
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.446724891662598
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.445246696472168
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.445833206176758
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.445683479309082
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.446892738342285
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.445212364196777
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.445100784301758
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.445877075195312
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.444957733154297
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.446231842041016
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.444456100463867
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.446710586547852
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.446854591369629
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.44576644897461
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.44590950012207
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.445233345031738
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.446449279785156
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.44591999053955
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.446203231811523
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.446487426757812
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.446499824523926
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.446240425109863
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.445124626159668
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.444564819335938
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.445398330688477
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.446983337402344
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.447197914123535
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.446784019470215
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.447357177734375
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.444705963134766
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.445945739746094
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.444546699523926
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.446733474731445
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.44636058807373
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.446362495422363
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.447315216064453
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.445897102355957
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.444729804992676
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.445178031921387
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.446642875671387
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.446335792541504
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.44752025604248
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.446934700012207
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.445253372192383
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.446026802062988
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.444671630859375
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.446354866027832
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.445853233337402
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.446110725402832
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.446598052978516
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.446369171142578
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.445677757263184
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.446175575256348
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.446813583374023
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.446487426757812
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.446715354919434
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.446258544921875
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.445032119750977
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.44625186920166
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.446349143981934
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.445685386657715
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.446003913879395
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.445815086364746
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.4470853805542
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.445157051086426
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.44617748260498
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.444929122924805
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.446699142456055
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.446615219116211
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.447568893432617
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.44766902923584
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.445507049560547
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.444254875183105
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.44710636138916
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.446497917175293
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.447943687438965
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.44632625579834
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.44657039642334
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.446557998657227
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.445124626159668
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.442789077758789
+                }
+            ]
+        },
+        {
+            "bit": 7,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.248947143554688
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.248910903930664
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.25033950805664
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.247231483459473
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.248175621032715
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.249284744262695
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.248127937316895
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.24847412109375
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.24928092956543
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.249184608459473
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.249013900756836
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.248451232910156
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.249009132385254
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.24847412109375
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.247840881347656
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.248675346374512
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.250819206237793
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.249013900756836
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.248690605163574
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.24866008758545
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.248364448547363
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.248971939086914
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.24841594696045
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.246623992919922
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.248726844787598
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.246759414672852
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.247727394104004
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.247196197509766
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.249139785766602
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.249351501464844
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.250386238098145
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.249241828918457
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.246981620788574
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.248429298400879
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.250593185424805
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.250153541564941
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.249406814575195
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.248666763305664
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.248573303222656
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.247270584106445
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.246830940246582
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.2464017868042
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.248993873596191
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.248223304748535
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.248842239379883
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.250232696533203
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.247780799865723
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.247939109802246
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.24886417388916
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.248420715332031
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.248710632324219
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.25039291381836
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.250648498535156
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.248642921447754
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.248636245727539
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.246983528137207
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.24928092956543
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.248407363891602
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.250875473022461
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.24842357635498
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.248676300048828
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.248442649841309
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.249731063842773
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.248982429504395
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.248353958129883
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.248805046081543
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.247725486755371
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.247814178466797
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.248486518859863
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.24753475189209
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.249300003051758
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.249649047851562
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.247969627380371
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.249044418334961
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.248712539672852
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.24815845489502
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.249027252197266
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.24854564666748
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.248103141784668
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.249629020690918
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.24824047088623
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.2481050491333
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.247129440307617
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.248519897460938
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.248733520507812
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.248607635498047
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.249269485473633
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.249617576599121
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.247703552246094
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.248675346374512
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.24885368347168
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.248747825622559
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.248342514038086
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.24862003326416
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.247286796569824
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.247939109802246
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.248085021972656
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.248418807983398
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.248696327209473
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.244585037231445
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.248316764831543
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.24746322631836
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.248014450073242
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.248686790466309
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.248486518859863
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.248899459838867
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.24615478515625
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.248085021972656
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.248323440551758
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.248067855834961
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.248703002929688
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.248798370361328
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.248700141906738
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.248745918273926
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.247742652893066
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.249890327453613
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.246377944946289
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.248982429504395
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.247485160827637
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.248905181884766
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.248494148254395
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.248453140258789
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.248908996582031
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.248163223266602
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.248038291931152
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.248063087463379
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.24870491027832
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.248576164245605
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.248796463012695
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.24866008758545
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.247627258300781
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.249587059020996
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.248126983642578
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.249279022216797
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.247621536254883
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.248908996582031
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.248923301696777
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.24789810180664
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.24841594696045
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.248531341552734
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.249009132385254
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.24869441986084
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.24914264678955
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.248514175415039
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.247979164123535
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.249058723449707
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.2473726272583
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.248682975769043
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.249214172363281
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.248796463012695
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.249458312988281
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.248420715332031
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.24854564666748
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.24852180480957
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.248820304870605
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.248818397521973
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.248915672302246
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.248651504516602
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.247900009155273
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.248698234558105
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.248372077941895
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.248966217041016
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.248855590820312
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.248828887939453
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.248640060424805
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.2474365234375
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.248297691345215
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.248397827148438
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.249364852905273
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.249395370483398
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.248762130737305
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.248749732971191
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.249072074890137
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.2484712600708
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.248058319091797
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.248920440673828
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.249072074890137
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.249566078186035
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.248096466064453
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.248063087463379
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.248846054077148
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.247840881347656
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.248753547668457
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.248805046081543
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.248598098754883
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.24842357635498
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.248464584350586
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.247590065002441
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.248529434204102
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.248451232910156
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.248435020446777
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.248573303222656
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.248891830444336
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.248395919799805
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.249190330505371
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.247916221618652
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.248773574829102
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.24884033203125
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.248529434204102
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.248580932617188
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.247758865356445
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.248266220092773
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.249462127685547
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.251774787902832
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.249398231506348
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.249236106872559
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.249151229858398
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.24839973449707
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.246936798095703
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.248584747314453
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.24909782409668
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.249333381652832
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.24841594696045
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.248838424682617
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.24821662902832
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.24967098236084
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.246960639953613
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.248788833618164
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.248041152954102
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.248600006103516
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.248079299926758
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.249293327331543
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.249005317687988
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.244156837463379
+                }
+            ]
+        },
+        {
+            "bit": 8,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.151074409484863
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.150999069213867
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.15017318725586
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.150079727172852
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.149909019470215
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.151076316833496
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.149611473083496
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.151737213134766
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.150666236877441
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.149551391601562
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.150126457214355
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.150810241699219
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.150899887084961
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.150544166564941
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.150005340576172
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.150588989257812
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.148330688476562
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.152186393737793
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.1503324508667
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.149892807006836
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.14985179901123
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.151497840881348
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.15092658996582
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.152995109558105
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.14924430847168
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.152137756347656
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.149636268615723
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.151265144348145
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.15049934387207
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.150126457214355
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.150693893432617
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.151034355163574
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.150919914245605
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.15013599395752
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.149358749389648
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.150907516479492
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.151817321777344
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.152956008911133
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.151284217834473
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.150101661682129
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.150654792785645
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.151396751403809
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.151276588439941
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.15075969696045
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.150779724121094
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.150075912475586
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.150604248046875
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.151717185974121
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.150860786437988
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.15180492401123
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.15007209777832
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.149370193481445
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.149572372436523
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.150145530700684
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.150029182434082
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.151187896728516
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.15129280090332
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.151187896728516
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.15044116973877
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.150922775268555
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.15063190460205
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.151873588562012
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.150771141052246
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.151354789733887
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.150899887084961
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.151061058044434
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.150907516479492
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.150270462036133
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.151175498962402
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.151531219482422
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.150693893432617
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.151203155517578
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.150079727172852
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.150582313537598
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.150604248046875
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.150514602661133
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.149866104125977
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.151334762573242
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.150800704956055
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.15058708190918
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.150219917297363
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.150744438171387
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.152250289916992
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.150511741638184
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.150794982910156
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.1510009765625
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.15099048614502
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.149415016174316
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.151323318481445
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.150444984436035
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.150690078735352
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.15068244934082
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.150378227233887
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.150261878967285
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.150903701782227
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.15018081665039
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.15124225616455
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.150435447692871
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.150575637817383
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.149094581604004
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.15013599395752
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.151214599609375
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.149642944335938
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.150808334350586
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.150075912475586
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.151111602783203
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.148834228515625
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.14988899230957
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.150835037231445
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.150402069091797
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.15018081665039
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.150197982788086
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.150830268859863
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.151409149169922
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.151313781738281
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.150514602661133
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.15072250366211
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.150216102600098
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.15139389038086
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.151105880737305
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.15129280090332
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.150542259216309
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.150880813598633
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.150386810302734
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.150701522827148
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.150864601135254
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.150973320007324
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.151409149169922
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.150779724121094
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.150169372558594
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.150347709655762
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.1502046585083
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.14988899230957
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.150887489318848
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.150087356567383
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.150806427001953
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.150779724121094
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.149798393249512
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.150814056396484
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.149948120117188
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.151045799255371
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.15125560760498
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.150981903076172
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.151280403137207
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.150588989257812
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.150777816772461
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.150565147399902
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.150872230529785
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.150402069091797
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.150662422180176
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.150643348693848
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.149776458740234
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.150899887084961
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.150068283081055
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.150911331176758
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.151187896728516
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.15097713470459
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.151018142700195
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.15003490447998
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.150360107421875
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.150002479553223
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.151078224182129
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.150967597961426
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.151137351989746
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.15079116821289
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.15003490447998
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.150252342224121
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.150211334228516
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.150823593139648
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.150959968566895
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.151315689086914
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.151352882385254
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.149739265441895
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.150032997131348
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.150887489318848
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.150884628295898
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.150869369506836
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.150604248046875
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.151308059692383
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.150044441223145
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.150025367736816
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.150693893432617
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.15110969543457
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.150965690612793
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.15166187286377
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.150911331176758
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.15001392364502
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.150763511657715
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.150779724121094
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.150981903076172
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.151012420654297
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.151111602783203
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.150527000427246
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.150915145874023
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.150227546691895
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.150995254516602
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.151082038879395
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.150897979736328
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.150961875915527
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.150580406188965
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.15087604522705
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.150973320007324
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.149886131286621
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.150259971618652
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.150981903076172
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.151164054870605
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.1516695022583
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.150505065917969
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.151898384094238
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.150216102600098
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.150959968566895
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.151091575622559
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.151031494140625
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.151062965393066
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.151004791259766
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.149847030639648
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.151618957519531
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.150933265686035
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.150798797607422
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.150670051574707
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.150973320007324
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.151222229003906
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.150849342346191
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.149383544921875
+                }
+            ]
+        },
+        {
+            "bit": 9,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.067623138427734
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.067371368408203
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.067314147949219
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.067713737487793
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.068294525146484
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.066909790039062
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.066911697387695
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.067405700683594
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.067975044250488
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.067959785461426
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.06697940826416
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.067327499389648
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.068471908569336
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.067337036132812
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.067325592041016
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.0684814453125
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.068489074707031
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.067191123962402
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.067529678344727
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.068572044372559
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.067832946777344
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.067422866821289
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.066040992736816
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.065824508666992
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.068029403686523
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.067051887512207
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.069037437438965
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.068551063537598
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.067594528198242
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.0676908493042
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.066457748413086
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.067981719970703
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.067009925842285
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.067184448242188
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.067936897277832
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.068052291870117
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.067959785461426
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.06750774383545
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.067174911499023
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.067588806152344
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.066601753234863
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.064970970153809
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.06764030456543
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.067378997802734
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.067084312438965
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.067852973937988
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.066695213317871
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.066423416137695
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.067400932312012
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.067340850830078
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.067607879638672
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.068846702575684
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.068791389465332
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.066926956176758
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.066832542419434
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.066413879394531
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.067556381225586
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.066998481750488
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.0679292678833
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.069302558898926
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.066902160644531
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.066278457641602
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.068281173706055
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.067770004272461
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.067181587219238
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.06659984588623
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.066482543945312
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.067448616027832
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.066686630249023
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.066936492919922
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.067852973937988
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.067567825317383
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.067138671875
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.068078994750977
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.06711196899414
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.067625045776367
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.067371368408203
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.067009925842285
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.067075729370117
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.066960334777832
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.067917823791504
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.06688117980957
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.066499710083008
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.067252159118652
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.06723690032959
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.067049980163574
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.066977500915527
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.066834449768066
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.066579818725586
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.067399024963379
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.06697940826416
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.06728458404541
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.067429542541504
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.066926956176758
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.06646728515625
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.066909790039062
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.067216873168945
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.067548751831055
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.066932678222656
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.0663480758667
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.067214012145996
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.066629409790039
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.066947937011719
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.067144393920898
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.067333221435547
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.066960334777832
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.066656112670898
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.067325592041016
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.066917419433594
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.06721019744873
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.067075729370117
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.067727088928223
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.067562103271484
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.06740951538086
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.066727638244629
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.067598342895508
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.065759658813477
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.067893028259277
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.06623649597168
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.067214012145996
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.067073822021484
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.06689453125
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.067201614379883
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.06643295288086
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.066864967346191
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.066125869750977
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.067594528198242
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.067240715026855
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.067051887512207
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.067164421081543
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.066367149353027
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.067806243896484
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.067405700683594
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.067583084106445
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.06662368774414
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.06707763671875
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.06748104095459
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.066814422607422
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.067150115966797
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.06710433959961
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.06735610961914
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.067261695861816
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.06727123260498
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.067252159118652
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.0669584274292
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.06726360321045
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.066512107849121
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.067444801330566
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.067535400390625
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.06706714630127
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.06736946105957
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.067084312438965
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.06692123413086
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.067275047302246
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.06718635559082
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.067146301269531
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.067079544067383
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.066890716552734
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.067407608032227
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.067298889160156
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.067463874816895
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.06735610961914
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.067179679870605
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.067066192626953
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.067422866821289
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.066814422607422
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.066925048828125
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.067325592041016
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.067655563354492
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.067460060119629
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.066898345947266
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.067273139953613
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.067323684692383
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.06699275970459
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.0665864944458
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.067240715026855
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.06734848022461
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.067789077758789
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.066925048828125
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.067155838012695
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.06695556640625
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.066548347473145
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.067193031311035
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.067556381225586
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.067394256591797
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.067628860473633
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.066964149475098
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.067400932312012
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.066832542419434
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.067021369934082
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.067121505737305
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.067450523376465
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.067978858947754
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.06725025177002
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.067359924316406
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.066597938537598
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.067267417907715
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.067283630371094
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.067208290100098
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.06723690032959
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.066777229309082
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.066756248474121
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.067429542541504
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.067161560058594
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.06722354888916
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.067436218261719
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.066812515258789
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.067388534545898
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.066712379455566
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.067301750183105
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.067394256591797
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.067215919494629
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.067553520202637
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.067460060119629
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.066627502441406
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.06711196899414
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.0663480758667
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.067488670349121
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.066911697387695
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.06745433807373
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.067044258117676
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.067448616027832
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.066147804260254
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.064809799194336
+                }
+            ]
+        },
+        {
+            "bit": 10,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 8.016072273254395
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 8.01572036743164
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 8.015779495239258
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 8.013850212097168
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 8.015554428100586
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 8.016152381896973
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 8.01573371887207
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 8.016106605529785
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 8.015592575073242
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8.014792442321777
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 8.015885353088379
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 8.015843391418457
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 8.01540756225586
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 8.016087532043457
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 8.01546859741211
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 8.016053199768066
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 8.01502513885498
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8.017158508300781
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 8.015349388122559
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 8.015369415283203
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 8.015707015991211
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 8.015777587890625
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 8.015740394592285
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 8.017233848571777
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 8.016221046447754
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 8.01547622680664
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 8.013744354248047
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 8.015820503234863
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 8.015275955200195
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 8.014629364013672
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 8.015417098999023
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 8.015705108642578
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 8.01575756072998
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 8.015128135681152
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 8.015640258789062
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 8.015661239624023
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 8.015885353088379
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 8.015984535217285
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 8.014740943908691
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 8.014925956726074
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 8.014516830444336
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 8.016642570495605
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 8.016022682189941
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 8.015360832214355
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 8.01651382446289
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 8.015082359313965
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8.015556335449219
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 8.016702651977539
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 8.014656066894531
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 8.016011238098145
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 8.01544189453125
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 8.014313697814941
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 8.014848709106445
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 8.015477180480957
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 8.015109062194824
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8.01556396484375
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 8.015725135803223
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 8.015539169311523
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 8.01572036743164
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 8.014037132263184
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 8.015162467956543
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 8.016515731811523
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 8.01517391204834
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 8.015525817871094
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 8.014971733093262
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8.016302108764648
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 8.016420364379883
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 8.015124320983887
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 8.015430450439453
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 8.015632629394531
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 8.015360832214355
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 8.015809059143066
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 8.015645980834961
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 8.015340805053711
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 8.015360832214355
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 8.015401840209961
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 8.015289306640625
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 8.01595401763916
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 8.015902519226074
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 8.015829086303711
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8.015710830688477
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 8.015657424926758
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 8.01676082611084
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 8.015522003173828
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 8.015560150146484
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 8.01569938659668
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 8.015657424926758
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 8.01547622680664
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 8.015851020812988
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 8.01551628112793
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 8.015908241271973
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 8.01565170288086
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 8.015670776367188
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 8.01548957824707
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8.014751434326172
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 8.0158052444458
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 8.015727996826172
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 8.015722274780273
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 8.015774726867676
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 8.014805793762207
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 8.015362739562988
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8.01598072052002
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 8.015769958496094
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 8.015583038330078
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 8.015472412109375
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 8.015750885009766
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 8.015069007873535
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 8.015493392944336
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 8.01559066772461
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 8.015027046203613
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 8.015701293945312
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 8.014756202697754
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 8.015510559082031
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 8.01571273803711
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 8.015560150146484
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 8.01537799835205
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 8.015793800354004
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 8.015349388122559
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 8.015571594238281
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 8.015885353088379
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 8.015772819519043
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 8.01509952545166
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 8.015621185302734
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 8.015426635742188
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 8.015628814697266
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 8.015556335449219
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 8.015602111816406
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 8.015594482421875
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 8.015811920166016
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 8.015523910522461
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 8.015731811523438
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 8.015135765075684
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 8.01469612121582
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 8.01568603515625
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 8.015399932861328
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 8.0157470703125
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 8.015573501586914
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 8.015280723571777
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 8.015426635742188
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 8.015299797058105
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 8.01565170288086
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 8.015868186950684
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 8.015632629394531
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 8.015829086303711
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 8.01514720916748
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 8.015705108642578
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 8.015910148620605
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 8.015628814697266
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 8.015725135803223
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 8.015820503234863
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 8.015647888183594
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 8.015191078186035
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 8.015379905700684
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 8.015149116516113
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 8.015727996826172
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 8.016009330749512
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 8.015762329101562
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 8.015332221984863
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 8.015636444091797
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 8.015419006347656
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 8.014669418334961
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 8.01570987701416
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 8.015827178955078
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 8.015898704528809
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 8.01575756072998
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 8.01526165008545
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 8.014975547790527
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 8.015481948852539
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 8.0158052444458
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 8.01585865020752
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 8.015950202941895
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 8.015764236450195
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 8.015149116516113
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 8.01553726196289
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 8.015166282653809
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 8.016026496887207
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 8.015839576721191
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 8.015560150146484
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 8.015844345092773
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 8.015491485595703
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 8.014870643615723
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 8.015875816345215
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 8.015854835510254
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 8.01578140258789
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 8.015820503234863
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 8.015586853027344
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 8.015535354614258
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 8.015332221984863
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 8.01523494720459
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 8.015959739685059
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 8.015628814697266
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 8.015625
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 8.015480041503906
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 8.015525817871094
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 8.015263557434082
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 8.016241073608398
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 8.015740394592285
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 8.015807151794434
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 8.01569938659668
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 8.015548706054688
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 8.01513957977295
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 8.015565872192383
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 8.015539169311523
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 8.015677452087402
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 8.015495300292969
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 8.01573371887207
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 8.015873908996582
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 8.014962196350098
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 8.015810012817383
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 8.014935493469238
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 8.015657424926758
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 8.015716552734375
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 8.016037940979004
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 8.01623249053955
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 8.0159330368042
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 8.016399383544922
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 8.015539169311523
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 8.015666961669922
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 8.015567779541016
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 8.015960693359375
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 8.015499114990234
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8.015135765075684
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 8.013542175292969
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 8.014952659606934
+                }
+            ]
+        },
+        {
+            "bit": 11,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.964605331420898
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.964717864990234
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.963918685913086
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.964454174041748
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.965520858764648
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.964694976806641
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.964081764221191
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.964750289916992
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.964765548706055
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.965912818908691
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.964085578918457
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.964970111846924
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.965302467346191
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.964750289916992
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.964552879333496
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.965578079223633
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.964978218078613
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.964252948760986
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.965638637542725
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.965236186981201
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.964364528656006
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.965023517608643
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.964137077331543
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.964049339294434
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.964387893676758
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.964309215545654
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.96549654006958
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.964499473571777
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.965106964111328
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.965099811553955
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.964205265045166
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.964746475219727
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.964237689971924
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.965942859649658
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.964859962463379
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.9645562171936035
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.965295314788818
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.965863227844238
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.965365886688232
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.9650163650512695
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.964442253112793
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.963918685913086
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.964932441711426
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.964480400085449
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.965057849884033
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.965120315551758
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.96401309967041
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.964643478393555
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.9651641845703125
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.9651713371276855
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.964643478393555
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.965460300445557
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.965175628662109
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.964601993560791
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.964146137237549
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.964431285858154
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.964890480041504
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.964850902557373
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.96538782119751
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.966098308563232
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.965152263641357
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.964096546173096
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.9650044441223145
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.964999198913574
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.964670658111572
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.964389801025391
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.964089393615723
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.9645256996154785
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.964934349060059
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.964175224304199
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.964991092681885
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.964664459228516
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.964712142944336
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.964696884155273
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.964609622955322
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.964184284210205
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.964624404907227
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.964731216430664
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.964516639709473
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.964993000030518
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.964263916015625
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.964488506317139
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.964402675628662
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.964738368988037
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.964560031890869
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.964544773101807
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.964890480041504
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.963897228240967
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.964389801025391
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.965316295623779
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.964731216430664
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.964503288269043
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.964643478393555
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.96470832824707
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.964254856109619
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.964338302612305
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.964414119720459
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.965084552764893
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.964207172393799
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.9640607833862305
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.964677810668945
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.964433193206787
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.9643707275390625
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.964776515960693
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.964890480041504
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.964840888977051
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.964601993560791
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.964417934417725
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.964118003845215
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.964818477630615
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.964771270751953
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.964954853057861
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.96466064453125
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.96486234664917
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.9643707275390625
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.964643478393555
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.96377420425415
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.964677810668945
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.964156150817871
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.964670658111572
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.964609622955322
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.964207172393799
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.964176177978516
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.964544773101807
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.964402675628662
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.963929653167725
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.9649739265441895
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.96478796005249
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.964738368988037
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.964704990386963
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.9644975662231445
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.964943885803223
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.964776515960693
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.964818477630615
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.964598178863525
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.9644694328308105
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.964754104614258
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.964231491088867
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.964653491973877
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.964601993560791
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.964852809906006
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.964600086212158
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.96473503112793
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.964757442474365
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.964710235595703
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.964892387390137
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.96381950378418
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.964693069458008
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.964879512786865
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.964605331420898
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.964568138122559
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.964784622192383
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.9649176597595215
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.965052604675293
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.964636325836182
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.9647955894470215
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.964563846588135
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.964765548706055
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.964274883270264
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.965055465698242
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.964712142944336
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.9648871421813965
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.96493673324585
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.964651584625244
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.964710235595703
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.964175224304199
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.964510917663574
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.964036464691162
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.964943885803223
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.964840888977051
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.964560031890869
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.964704990386963
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.964919567108154
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.9641499519348145
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.96450138092041
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.964621067047119
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.964864253997803
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.964890480041504
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.964803695678711
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.964305877685547
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.96493673324585
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.9647955894470215
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.9645490646362305
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.964954853057861
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.96500301361084
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.964771270751953
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.964423656463623
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.964552879333496
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.964328289031982
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.964636325836182
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.964684009552002
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.9645490646362305
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.9649739265441895
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.964776515960693
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.964719772338867
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.964321136474609
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.964856147766113
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.964503288269043
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.965101718902588
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.964563846588135
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.964305877685547
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.965035438537598
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.964750289916992
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.96473503112793
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.964820861816406
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.96476936340332
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.964812755584717
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.964561939239502
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.964465141296387
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.964831829071045
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.964761257171631
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.964622497558594
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.964542865753174
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.964523792266846
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.96421480178833
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.964547157287598
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.964442253112793
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.96466064453125
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.96444034576416
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.96466064453125
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.964396953582764
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.96507453918457
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.963815689086914
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.962518692016602
+                }
+            ]
+        },
+        {
+            "bit": 12,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.934129238128662
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.93372106552124
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.936017990112305
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.933906555175781
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.933186054229736
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.93415641784668
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.933126926422119
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.934360027313232
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.9343109130859375
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.933248519897461
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.93372106552124
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.93415641784668
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.933910369873047
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.933342933654785
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.933637619018555
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.934277057647705
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.9337592124938965
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.93411111831665
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.934088230133057
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.9338154792785645
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.933320045471191
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.933942794799805
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.934088230133057
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.934545516967773
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.93414831161499
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.93388557434082
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.933516979217529
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.9342546463012695
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.933891296386719
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.933448791503906
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.934754371643066
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.934265613555908
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.93405818939209
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.932851314544678
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.934825897216797
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.934067249298096
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.933448791503906
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.933569431304932
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.93362283706665
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.932837963104248
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.933821201324463
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.934160232543945
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.934116363525391
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.934638977050781
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.933763027191162
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.933970928192139
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.934314727783203
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.93358850479126
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.9337968826293945
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.934328079223633
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.93355655670166
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.933483123779297
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.9330668449401855
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.933656692504883
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.9337286949157715
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.934144973754883
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.93381404876709
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.93385124206543
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.9332404136657715
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.933043956756592
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.9338483810424805
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.934352874755859
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.93358850479126
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.933749198913574
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.93378210067749
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.934314727783203
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.93448543548584
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.93405818939209
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.933607578277588
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.934072971343994
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.934086322784424
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.934205532073975
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.933509349822998
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.933671951293945
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.933773994445801
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.933562278747559
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.933722972869873
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.934152603149414
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.93407678604126
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.933411121368408
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.933831214904785
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.934088230133057
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.93415641784668
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.933518409729004
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.933898448944092
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.934039115905762
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.933770179748535
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.93420934677124
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.934046268463135
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.933117866516113
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.934055805206299
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.934025764465332
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.934263706207275
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.933863162994385
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.9343037605285645
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.933955669403076
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.934280872344971
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.934051990509033
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.934103012084961
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.933267593383789
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.933906555175781
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.933857440948486
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.9337477684021
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.93408203125
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.9338765144348145
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.93388557434082
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.933732032775879
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.933959484100342
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.934375762939453
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.933607578277588
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.9340009689331055
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.933936595916748
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.933838367462158
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.934141159057617
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.9336419105529785
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.933942794799805
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.934241771697998
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.933488368988037
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.934354782104492
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.93415641784668
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.934002876281738
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.93383264541626
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.934013843536377
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.93349027633667
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.933881759643555
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.933910369873047
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.933732032775879
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.934247016906738
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.933902740478516
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.934067249298096
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.934109210968018
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.933754920959473
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.933513164520264
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.9339728355407715
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.934103012084961
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.933857440948486
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.933985710144043
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.93355655670166
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.934025764465332
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.933645725250244
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.934025764465332
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.934146404266357
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.934042930603027
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.933993816375732
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.933691024780273
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.933630466461182
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.9337358474731445
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.9341583251953125
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.9338765144348145
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.934021949768066
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.933887481689453
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.933367729187012
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.934190273284912
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.933584690093994
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.934046268463135
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.934051990509033
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.934093952178955
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.93409538269043
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.933821201324463
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.9339728355407715
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.93345832824707
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.933955669403076
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.933974742889404
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.934210777282715
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.933963775634766
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.933724880218506
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.933615684509277
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.934122085571289
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.934250831604004
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.934013843536377
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.9343719482421875
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.9341864585876465
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.933637619018555
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.9338836669921875
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.934178352355957
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.934356689453125
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.934118270874023
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.93405818939209
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.934430122375488
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.933948040008545
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.9332404136657715
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.933861255645752
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.9340128898620605
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.9339447021484375
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.933919429779053
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.933906555175781
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.933831214904785
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.934034824371338
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.933913707733154
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.934175491333008
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.933849334716797
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.934292793273926
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.93391752243042
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.933679580688477
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.933710098266602
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.934432506561279
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.934114456176758
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.934210777282715
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.933777809143066
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.934093952178955
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.933834552764893
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.93364953994751
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.933694839477539
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.934201717376709
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.934079170227051
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.934231758117676
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.933955669403076
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.933682918548584
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.933953762054443
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.933509349822998
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.934273719787598
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.934116363525391
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.934192180633545
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.934500694274902
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.934457302093506
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.933518409729004
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.933679580688477
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.934322834014893
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.933902740478516
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.93407678604126
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.934103012084961
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.933732032775879
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.933861255645752
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.933682918548584
+                }
+            ]
+        },
+        {
+            "bit": 13,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.8956828117370605
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.89573335647583
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.894374370574951
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.8953046798706055
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.89564323425293
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.895830154418945
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.896002769470215
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.895630359649658
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.895513534545898
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.8961687088012695
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.896307468414307
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.895694255828857
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.896726131439209
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.895764350891113
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.895650863647461
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.895998954772949
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.896126747131348
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.895295143127441
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.895564556121826
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.896631717681885
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.8966169357299805
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.895965099334717
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.895095348358154
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.894772052764893
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.895986080169678
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.895052433013916
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.89622163772583
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.895277976989746
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.895852088928223
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.895869255065918
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.895908832550049
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.896046161651611
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.895775318145752
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.896175861358643
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.895323276519775
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.895368576049805
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.896428108215332
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.896627902984619
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.895649433135986
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.8954010009765625
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.895233154296875
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.895171165466309
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.8955607414245605
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.895434856414795
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.895754337310791
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.8963117599487305
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.895739555358887
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.896117687225342
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.895961761474609
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.895698070526123
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.895498752593994
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.896678924560547
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.8964362144470215
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.8955206871032715
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.895924091339111
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.895174503326416
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.895956039428711
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.895362854003906
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.896251678466797
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.897162437438965
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.895675182342529
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.895190238952637
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.895645618438721
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.89582633972168
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.895641326904297
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.895571708679199
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.895076274871826
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.895146369934082
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.895658493041992
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.895318031311035
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.895584583282471
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.895481586456299
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.895583629608154
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.895359039306641
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.895360946655273
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.895660400390625
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.895846843719482
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.895596504211426
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.8954925537109375
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.89573335647583
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.895809173583984
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.895403861999512
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.895298957824707
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.895803451538086
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.895381927490234
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.8954267501831055
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.895140647888184
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.894763946533203
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.894922733306885
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.8962249755859375
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.895295143127441
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.895748615264893
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.895641326904297
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.895782470703125
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.8951334953308105
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.895419597625732
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.895370960235596
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.895479679107666
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.895403861999512
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.89509916305542
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.895415782928467
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.895652770996094
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.89562463760376
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.89539098739624
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.895415782928467
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.895551681518555
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.895366668701172
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.895396709442139
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.895470142364502
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.896060943603516
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.895468711853027
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.895201206207275
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.895799160003662
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.895442008972168
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.895609378814697
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.895637512207031
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.894946575164795
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.896158695220947
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.894984722137451
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.895528793334961
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.895724296569824
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.895240306854248
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.8952460289001465
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.895235061645508
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.895737171173096
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.894929885864258
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.895430564880371
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.895541667938232
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.895357131958008
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.895231246948242
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.894967555999756
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.895703315734863
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.89561128616333
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.895503997802734
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.895522594451904
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.895366668701172
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.8955206871032715
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.895242214202881
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.895577430725098
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.895434856414795
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.895660400390625
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.895517349243164
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.895600318908691
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.895502090454102
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.895242214202881
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.895750522613525
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.895174503326416
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.895199298858643
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.89581823348999
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.895662307739258
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.895430564880371
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.895660400390625
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.895056247711182
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.895852088928223
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.895652770996094
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.895622730255127
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.8951520919799805
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.895498752593994
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.89562463760376
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.8953399658203125
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.8961310386657715
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.895577430725098
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.895566463470459
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.895176410675049
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.895370960235596
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.894986629486084
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.895667552947998
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.895009517669678
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.895488739013672
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.895630359649658
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.895308017730713
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.895758152008057
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.89570951461792
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.895498752593994
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.895359039306641
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.895628452301025
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.895605564117432
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.895637512207031
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.895543575286865
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.895242214202881
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.895820140838623
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.895549297332764
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.895566463470459
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.895761966705322
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.895822048187256
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.895502090454102
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.895486831665039
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.895569801330566
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.895860195159912
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.895483493804932
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.895901203155518
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.895679473876953
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.8956074714660645
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.8958940505981445
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.895456790924072
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.895295143127441
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.8954267501831055
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.895562648773193
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.89567756652832
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.895632266998291
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.895359039306641
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.89578104019165
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.895551681518555
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.896018028259277
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.895411968231201
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.895556926727295
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.895667552947998
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.895573616027832
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.895295143127441
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.89584493637085
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.895503997802734
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.895509719848633
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.895528793334961
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.895442008972168
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.895163536071777
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.895954132080078
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.895790100097656
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.895615577697754
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.895419597625732
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.895901203155518
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.895490646362305
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.896251678466797
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.895005226135254
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.894482135772705
+                }
+            ]
+        },
+        {
+            "bit": 14,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.880334854125977
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.88006067276001
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.880678653717041
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.880212783813477
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.879787921905518
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.880086898803711
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.879388332366943
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.880111217498779
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.880492687225342
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.879292011260986
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.879976272583008
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.880184650421143
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.879919528961182
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.880273342132568
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.88018274307251
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.880095958709717
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.879842758178711
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.880460739135742
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.880008220672607
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.879767417907715
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.8791961669921875
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.8802971839904785
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.879979610443115
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.881012916564941
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.880107879638672
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.880434513092041
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.8791890144348145
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.880094051361084
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.879912376403809
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.880113124847412
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.880224227905273
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.8803181648254395
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.879900932312012
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.879220962524414
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.879660606384277
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.880471706390381
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.880030155181885
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.8806939125061035
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.879403114318848
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.880186557769775
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.8806352615356445
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.880791664123535
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.880276203155518
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.8806538581848145
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.880389213562012
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.879976272583008
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.879840850830078
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.880631923675537
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.879641532897949
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.880451679229736
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.8797502517700195
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.878563404083252
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.8796305656433105
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.879915714263916
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.8796844482421875
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.879904747009277
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.88001012802124
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.880090713500977
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.8796563148498535
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.879581451416016
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.8802056312561035
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.880757808685303
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.880280494689941
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.87990665435791
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.88039493560791
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.880056858062744
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.880690097808838
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.8799028396606445
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.8799614906311035
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.880396842956543
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.879998207092285
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.8802900314331055
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.879422187805176
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.880302906036377
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.88018274307251
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.879996299743652
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.8795270919799805
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.8802900314331055
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.880186557769775
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.879912376403809
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.879652500152588
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.880186557769775
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.880644798278809
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.879812717437744
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.880430698394775
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.880173683166504
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.880256175994873
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.880690097808838
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.880320072174072
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.879589080810547
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.880148887634277
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.880032062530518
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.880227565765381
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.880280494689941
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.879776954650879
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.880086898803711
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.880116939544678
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.880184650421143
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.880321979522705
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.8799872398376465
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.880113124847412
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.880111217498779
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.880088806152344
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.880058765411377
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.8798747062683105
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.880083084106445
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.879889488220215
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.880099773406982
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.880468368530273
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.8800249099731445
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.880207538604736
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.880359172821045
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.879851818084717
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.880280494689941
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.879707336425781
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.879946231842041
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.880173683166504
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.8800740242004395
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.880175590515137
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.880210876464844
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.879929542541504
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.880163669586182
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.880273342132568
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.88009786605835
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.880107879638672
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.8802056312561035
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.880302906036377
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.880299091339111
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.880077838897705
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.880596160888672
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.880116939544678
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.880154609680176
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.879668235778809
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.880242347717285
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.8803181648254395
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.880197525024414
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.880141735076904
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.879796028137207
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.880192279815674
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.8799614906311035
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.880320072174072
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.880332946777344
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.880307197570801
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.8803629875183105
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.87985372543335
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.880141735076904
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.880126476287842
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.880302906036377
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.879964351654053
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.880246639251709
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.880133628845215
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.879915714263916
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.880152702331543
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.879900932312012
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.880235195159912
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.880332946777344
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.88004732131958
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.880288124084473
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.880094051361084
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.880321979522705
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.879859924316406
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.880184650421143
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.880111217498779
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.88018274307251
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.880171775817871
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.8800811767578125
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.879889488220215
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.880235195159912
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.880288124084473
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.880133628845215
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.8803558349609375
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.880610942840576
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.879991054534912
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.880197525024414
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.879972457885742
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.880382061004639
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.880382061004639
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.880282402038574
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.88029146194458
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.880246639251709
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.879754066467285
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.880250453948975
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.880026817321777
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.880145072937012
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.880167484283447
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.880327224731445
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.88020133972168
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.880143642425537
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.879705429077148
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.880107879638672
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.8800811767578125
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.88020133972168
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.880127906799316
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.880321979522705
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.880099773406982
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.88029146194458
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.880364894866943
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.880343914031982
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.879992961883545
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.8800811767578125
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.880017280578613
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.880258083343506
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.880012035369873
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.8802714347839355
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.880458831787109
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.880407810211182
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.880453586578369
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.879848480224609
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.880195617675781
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.879605770111084
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.880407810211182
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.880332946777344
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.880640983581543
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.880573272705078
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.880295276641846
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.8800249099731445
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.8803558349609375
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.880305290222168
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.8803839683532715
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.879970550537109
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.88018274307251
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.880083084106445
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.880154609680176
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.879299640655518
+                }
+            ]
+        },
+        {
+            "bit": 15,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.851803779602051
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.851534843444824
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.851141452789307
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.851541996002197
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.852069854736328
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.851837635040283
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.851553440093994
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.85181188583374
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.851686477661133
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.8524909019470215
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.851568222045898
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.85197639465332
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.852665424346924
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.851968765258789
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.851800918579102
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.85217809677124
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.851399898529053
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.85080623626709
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.852111339569092
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.851820945739746
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.853028297424316
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.851682662963867
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.851198196411133
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.850396633148193
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.851465225219727
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.851661682128906
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.851912975311279
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.851450443267822
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.851893901824951
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.851579189300537
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.850673675537109
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.851841449737549
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.851587295532227
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.852566242218018
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.852047443389893
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.851577281951904
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.852357864379883
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.852042198181152
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.8522047996521
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.851201057434082
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.850250720977783
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.850669860839844
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.85175895690918
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.851264953613281
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.852581024169922
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.85195779800415
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.851490020751953
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.851355075836182
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.852105617523193
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.851989269256592
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.851344108581543
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.852625846862793
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.852175235748291
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.851635932922363
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.851373672485352
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.851502895355225
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.8521223068237305
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.851395606994629
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.85249662399292
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.852126121520996
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.8512725830078125
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.8511433601379395
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.851635932922363
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.851790904998779
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.85185432434082
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.851107597351074
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.851478099822998
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.851834297180176
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.851776123046875
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.851751804351807
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.851510047912598
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.851768970489502
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.851160526275635
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.85174036026001
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.851614952087402
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.85175895690918
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.851901054382324
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.851712226867676
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.851632118225098
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.85183048248291
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.851901054382324
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.851733207702637
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.851454257965088
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.851933002471924
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.851674556732178
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.851592540740967
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.851182460784912
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.850868225097656
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.851665496826172
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.851873397827148
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.85175895690918
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.851669788360596
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.8516845703125
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.851369857788086
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.851444244384766
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.851452350616455
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.851781845092773
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.850999355316162
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.851558685302734
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.851351261138916
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.851418495178223
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.851422309875488
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.851661682128906
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.851663589477539
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.851762771606445
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.851757049560547
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.851639270782471
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.851624011993408
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.8516526222229
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.851980209350586
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.851414680480957
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.851186275482178
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.85186767578125
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.851674556732178
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.851508140563965
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.851630210876465
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.851328372955322
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.851654529571533
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.851721286773682
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.851672649383545
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.851824760437012
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.8512349128723145
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.851358890533447
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.851388931274414
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.851497173309326
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.851109504699707
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.851579189300537
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.851737022399902
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.851966857910156
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.851499080657959
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.851687908172607
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.851787090301514
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.8519744873046875
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.851686477661133
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.851699352264404
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.851418495178223
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.851437568664551
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.851767063140869
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.851837635040283
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.851572513580322
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.851714134216309
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.851837635040283
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.85183572769165
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.851737022399902
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.851938724517822
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.851998329162598
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.851687908172607
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.851570129394531
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.851674556732178
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.851616859436035
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.851815700531006
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.851566314697266
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.85153865814209
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.851866245269775
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.851650714874268
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.851646900177002
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.851729393005371
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.851805686950684
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.851399898529053
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.851564407348633
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.851678848266602
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.851800918579102
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.851672649383545
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.851480007171631
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.851749897003174
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.851306438446045
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.852004528045654
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.851478099822998
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.851747989654541
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.85174036026001
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.8517351150512695
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.851785659790039
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.851470947265625
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.851229667663574
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.851358890533447
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.851624011993408
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.851800918579102
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.851800918579102
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.85175895690918
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.851785659790039
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.852242469787598
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.8516526222229
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.85155725479126
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.85175895690918
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.851934909820557
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.851843357086182
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.8514275550842285
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.851532936096191
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.852152347564697
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.851849555969238
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.851785659790039
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.851898193359375
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.851841449737549
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.85195779800415
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.8519182205200195
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.851534843444824
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.851659774780273
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.8517045974731445
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.85186243057251
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.8515944480896
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.851792812347412
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.851499080657959
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.8518829345703125
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.851834297180176
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.851663589477539
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.851680755615234
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.851504802703857
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.851819038391113
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.851916790008545
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.851949691772461
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.851828575134277
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.851654529571533
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.851800918579102
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.851768970489502
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.851674556732178
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.85249662399292
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.851317405700684
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.851624011993408
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.851463317871094
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.8518452644348145
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.851373672485352
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.851847171783447
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.850452423095703
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.8510332107543945
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file