diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,13520 @@ +{ + "original_ppl": 6.907054424285889, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 57994.33203125 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 57655.12890625 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 46566.56640625 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 44251.4453125 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 52224.15625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 54722.71484375 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 57537.30859375 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 61086.91015625 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 64757.16796875 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 126668.3671875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 55783.19140625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 61520.59375 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 48151.71875 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 65466.16796875 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 49455.11328125 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 47864.37890625 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 83822.3046875 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 74297.71875 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 46040.20703125 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 60308.12890625 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 63651.625 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 56727.61328125 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 57271.30859375 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 96543.3046875 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 72228.1953125 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 77999.4296875 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 64231.77734375 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 90974.8828125 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 61700.56640625 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 58228.3671875 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 59047.43359375 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 69410.8515625 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 51735.23046875 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 54218.5234375 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 64930.625 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 59848.96875 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 56517.9296875 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 49421.16796875 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 90543.5 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 62434.6640625 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 49555.390625 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 65460.234375 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 59721.87890625 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 62125.09375 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 43092.5078125 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 69688.6328125 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 62992.0703125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 62623.6953125 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 45661.1875 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 63618.125 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 61360.8671875 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 70915.1328125 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 90062.265625 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 98518.4609375 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 58045.34765625 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 56417.7109375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 52509.8125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 54299.96875 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 50958.66015625 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 58610.671875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 63032.875 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 56921.40234375 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 47526.01171875 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 56775.0234375 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 59230.6796875 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 74920.5859375 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 62319.07421875 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 64058.9609375 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 43275.36328125 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 65219.2109375 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 57287.19921875 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 69236.5859375 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 62028.71484375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 69365.921875 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 72758.875 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 46378.1640625 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 50492.21875 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 56953.328125 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 56917.11328125 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 47963.53515625 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 81038.3046875 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 85742.109375 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 44483.86328125 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 64872.7578125 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 58727.94140625 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 55757.5 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 71045.7734375 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 79875.859375 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 61395.9296875 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 49989.8515625 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 55269.48828125 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 59208.19921875 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 50609.2265625 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 63704.2734375 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 55155.12109375 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 70248.2734375 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 40381.3359375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 51299.19921875 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 57204.2734375 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 56551.0859375 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 46413.6953125 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 38129.37890625 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 56092.6015625 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 62208.33203125 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 58819.1953125 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 57401.28125 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 54386.671875 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 74688.671875 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 50139.25 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 57336.50390625 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 53037.1015625 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 53838.77734375 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 54780.5703125 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 51946.8828125 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 63058.90625 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 55205.16796875 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 64622.80078125 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 46986.1640625 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 55213.85546875 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 58540.28515625 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 54732.52734375 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 73575.6953125 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 58874.359375 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 61122.28515625 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 60138.2421875 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 55670.6328125 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 58418.0390625 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 57677.4609375 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 62802.76171875 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 58413.96875 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 66505.7265625 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 51112.0703125 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 54442.98046875 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 58896.37890625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 58653.94921875 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 57087.265625 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 53299.1484375 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 55671.42578125 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 57696.4921875 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 61450.11328125 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 58204.05078125 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 58612.57421875 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 61778.171875 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 53000.34375 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 59795.85546875 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 49785.26953125 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 60216.8046875 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 60638.18359375 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 58789.80859375 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 59496.8203125 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 57087.21484375 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 60533.83203125 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 59070.80859375 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 57660.62890625 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 58898.00390625 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 58710.41796875 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 59580.96484375 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 63309.45703125 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 56452.37109375 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 57827.046875 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 58046.01171875 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 58366.859375 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 58329.85546875 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 58310.890625 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 54671.59375 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 61638.93359375 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 55567.83203125 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 60331.9453125 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 58658.42578125 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 58035.4375 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 51761.6796875 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 59279.1640625 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 58931.32421875 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 57300.48046875 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 60910.53125 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 58609.44140625 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 59316.31640625 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 61136.5078125 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 56925.36328125 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 57281.02734375 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 55192.32421875 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 62393.70703125 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 58919.015625 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 57761.34765625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 64054.3828125 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 57518.984375 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 56464.109375 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 60748.15625 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 56556.80859375 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 59941.1015625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 59675.875 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 57500.83203125 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 61255.09765625 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 58521.0859375 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 63087.05859375 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 61676.09375 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 59302.23046875 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 57760.35546875 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 60390.14453125 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 58421.046875 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 56302.1015625 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 56961.53125 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 60580.0390625 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 60015.23046875 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 53553.8515625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 55395.34375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 54341.828125 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 53754.125 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 60077.765625 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 53521.9921875 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 59385.0859375 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 62964.44140625 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 55486.96875 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 70458.8828125 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 42390.53125 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 63289.35546875 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 57030.45703125 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 58814.98828125 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 61300.26953125 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 54327.94140625 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 68645.390625 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 52679.44921875 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 76306.234375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 47326.32421875 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 206.385009765625 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 206.25633239746094 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 210.4254913330078 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 197.64779663085938 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 193.6863555908203 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 195.6606903076172 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 185.86390686035156 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 201.95066833496094 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 204.22093200683594 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 195.4046630859375 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 188.76617431640625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 196.41909790039062 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 192.04661560058594 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 195.8568115234375 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 167.5603485107422 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 161.82606506347656 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 214.33604431152344 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 206.35037231445312 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 196.1558837890625 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 192.66612243652344 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 192.29916381835938 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 201.63958740234375 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 196.80075073242188 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 205.38307189941406 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 205.926025390625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 198.71621704101562 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 206.09500122070312 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 191.90756225585938 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 203.48355102539062 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 202.01173400878906 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 205.55450439453125 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 194.0542755126953 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 189.86624145507812 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 195.68606567382812 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 195.49049377441406 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 198.92800903320312 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 209.11485290527344 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 201.74575805664062 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 189.7243194580078 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 200.5698699951172 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 200.8448486328125 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 197.94810485839844 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 199.19284057617188 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 202.46893310546875 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 201.55970764160156 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 193.3639373779297 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 197.5482940673828 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 200.147216796875 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 196.3020477294922 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 199.07159423828125 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 200.41940307617188 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 202.97613525390625 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 200.85585021972656 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 198.44029235839844 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 191.14508056640625 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 203.42825317382812 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 206.77035522460938 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 202.14337158203125 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 209.67088317871094 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 211.37925720214844 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 202.70144653320312 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 204.64244079589844 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 197.88157653808594 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 203.33291625976562 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 201.67063903808594 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 197.65957641601562 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 189.52024841308594 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 199.50634765625 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 197.1781768798828 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 198.95069885253906 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 202.98912048339844 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 205.15127563476562 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 200.31411743164062 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 196.54603576660156 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 199.41778564453125 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 206.91188049316406 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 200.616455078125 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 197.63233947753906 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 200.83804321289062 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 202.7667999267578 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 194.49951171875 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 202.66299438476562 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 202.70028686523438 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 201.58670043945312 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 205.6596221923828 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 204.2587127685547 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 192.60511779785156 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 193.5184326171875 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 201.78472900390625 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 201.3412628173828 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 204.80313110351562 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 204.48812866210938 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 202.4530029296875 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 199.81155395507812 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 203.35842895507812 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 197.31895446777344 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 207.17718505859375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 208.383544921875 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 204.43870544433594 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 200.06686401367188 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 199.68431091308594 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 199.64898681640625 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 200.29615783691406 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 193.34400939941406 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 202.27671813964844 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 201.27090454101562 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 200.93536376953125 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 204.79005432128906 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 200.72698974609375 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 200.7106170654297 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 201.14991760253906 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 199.3990478515625 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 202.38641357421875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 203.7515106201172 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 199.41265869140625 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 199.0663604736328 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 200.66323852539062 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 204.3282928466797 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 202.0742645263672 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 202.7115936279297 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 203.2186279296875 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 202.7713623046875 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 202.13893127441406 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 203.68527221679688 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 201.63400268554688 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 199.4534454345703 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 203.43154907226562 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 204.250732421875 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 205.69374084472656 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 203.79835510253906 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 203.5410919189453 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 201.5070343017578 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 204.04095458984375 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 205.61666870117188 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 203.8982696533203 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 206.3537139892578 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 204.06809997558594 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 204.09068298339844 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 203.1705780029297 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 203.12989807128906 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 204.31503295898438 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 204.83633422851562 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 205.23757934570312 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 204.85089111328125 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 203.62759399414062 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 207.9664306640625 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 203.41351318359375 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 204.5959930419922 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 204.20437622070312 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 204.50169372558594 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 202.42337036132812 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 203.37228393554688 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 203.70140075683594 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 204.18228149414062 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 205.2613525390625 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 204.51632690429688 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 205.4357452392578 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 204.4656982421875 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 204.51026916503906 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 204.15707397460938 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 203.50665283203125 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 204.6703643798828 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 204.73770141601562 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 204.19232177734375 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 203.19277954101562 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 203.7767791748047 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 204.64117431640625 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 204.74864196777344 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 204.03221130371094 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 204.6879119873047 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 205.30178833007812 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 203.84024047851562 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 204.33958435058594 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 205.75857543945312 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 204.255615234375 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 203.91868591308594 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 203.8014678955078 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 203.80165100097656 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 201.73046875 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 203.72860717773438 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 203.48666381835938 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 205.48963928222656 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 204.51182556152344 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 204.6520233154297 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 206.28858947753906 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 203.47422790527344 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 204.33529663085938 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 204.8705291748047 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 204.55972290039062 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 203.4470672607422 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 203.82070922851562 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 204.85400390625 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 204.9785919189453 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 205.79498291015625 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 206.72235107421875 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 205.19451904296875 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 204.60536193847656 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 204.92152404785156 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 205.1990203857422 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 204.55113220214844 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 203.88272094726562 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 204.488525390625 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 204.4867706298828 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 206.19259643554688 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 204.22093200683594 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 205.43370056152344 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 203.97344970703125 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 203.9443817138672 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 206.84373474121094 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 204.9292449951172 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 204.3677520751953 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 205.77212524414062 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 203.94203186035156 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 205.16282653808594 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 206.9185028076172 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 205.52835083007812 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 204.8912353515625 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 206.1145477294922 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 204.45118713378906 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 208.5819549560547 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 202.28529357910156 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 196.20162963867188 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 198.00927734375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 192.1659698486328 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 11.370417594909668 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 11.368903160095215 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 11.353796005249023 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 11.325138092041016 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 11.356973648071289 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 11.34092903137207 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 11.344145774841309 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 11.344606399536133 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 11.356893539428711 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 11.343437194824219 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 11.31881046295166 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 11.341134071350098 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 11.31543254852295 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 11.329577445983887 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 11.307307243347168 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 11.304469108581543 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 11.358509063720703 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 11.349102020263672 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 11.324419975280762 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 11.34676456451416 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 11.323474884033203 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 11.355037689208984 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 11.332587242126465 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 11.357710838317871 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 11.30777359008789 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 11.328686714172363 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 11.355826377868652 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 11.337882041931152 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 11.358301162719727 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 11.346299171447754 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 11.35077953338623 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 11.3433837890625 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 11.292462348937988 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 11.324525833129883 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 11.328229904174805 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 11.353062629699707 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 11.373270034790039 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 11.370201110839844 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 11.382454872131348 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 11.326525688171387 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 11.368596076965332 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 11.347148895263672 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 11.356443405151367 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 11.365189552307129 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 11.324743270874023 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 11.344674110412598 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 11.323004722595215 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 11.36799430847168 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 11.323269844055176 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 11.342748641967773 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 11.335349082946777 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 11.344557762145996 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 11.35498332977295 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 11.338532447814941 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 11.3262939453125 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 11.335640907287598 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 11.398869514465332 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 11.371480941772461 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 11.353099822998047 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 11.381402015686035 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 11.339682579040527 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 11.340079307556152 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 11.359237670898438 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 11.352273941040039 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 11.353654861450195 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 11.375770568847656 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 11.363875389099121 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 11.317879676818848 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 11.357152938842773 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 11.348041534423828 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 11.390130043029785 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 11.392908096313477 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 11.370721817016602 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 11.356215476989746 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 11.355817794799805 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 11.377635955810547 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 11.368181228637695 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 11.366525650024414 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 11.367105484008789 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 11.369138717651367 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 11.351518630981445 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 11.348627090454102 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 11.367593765258789 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 11.339262962341309 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 11.362896919250488 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 11.372830390930176 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 11.34312915802002 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 11.360234260559082 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 11.32846450805664 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 11.357453346252441 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 11.355841636657715 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 11.36800479888916 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 11.366934776306152 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 11.376307487487793 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 11.342352867126465 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 11.35086727142334 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 11.367514610290527 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 11.361800193786621 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 11.367767333984375 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 11.366297721862793 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 11.350390434265137 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 11.361567497253418 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 11.357933044433594 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 11.350836753845215 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 11.347259521484375 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 11.369034767150879 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 11.350439071655273 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 11.378312110900879 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 11.35106086730957 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 11.35019302368164 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 11.357624053955078 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 11.346668243408203 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 11.370027542114258 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 11.376876831054688 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 11.36391830444336 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 11.36309814453125 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 11.33577823638916 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 11.365482330322266 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 11.346848487854004 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 11.374110221862793 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 11.369775772094727 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 11.36721420288086 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 11.362997055053711 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 11.359139442443848 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 11.362829208374023 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 11.348220825195312 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 11.376100540161133 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 11.374815940856934 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 11.364428520202637 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 11.373907089233398 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 11.355119705200195 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 11.375105857849121 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 11.356232643127441 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 11.380972862243652 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 11.376139640808105 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 11.385307312011719 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 11.374427795410156 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 11.361414909362793 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 11.364355087280273 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 11.356280326843262 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 11.376041412353516 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 11.377178192138672 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 11.37439250946045 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 11.37668228149414 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 11.354716300964355 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 11.370889663696289 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 11.360369682312012 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 11.373289108276367 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 11.377893447875977 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 11.368943214416504 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 11.374105453491211 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 11.355582237243652 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 11.364384651184082 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 11.349971771240234 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 11.379573822021484 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 11.372784614562988 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 11.372687339782715 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 11.369344711303711 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 11.356783866882324 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 11.363263130187988 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 11.355273246765137 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 11.37451171875 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 11.375764846801758 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 11.374739646911621 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 11.371197700500488 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 11.360727310180664 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 11.364324569702148 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 11.360435485839844 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 11.37796688079834 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 11.377387046813965 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 11.377156257629395 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 11.38155460357666 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 11.36296558380127 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 11.373058319091797 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 11.360055923461914 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 11.374017715454102 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 11.366466522216797 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 11.36633586883545 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 11.373937606811523 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 11.363260269165039 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 11.3628568649292 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 11.366514205932617 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 11.3713960647583 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 11.371500015258789 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 11.382357597351074 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 11.369999885559082 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 11.362415313720703 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 11.360952377319336 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 11.365766525268555 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 11.370933532714844 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 11.379313468933105 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 11.374929428100586 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 11.368921279907227 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 11.366742134094238 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 11.371052742004395 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 11.369246482849121 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 11.373015403747559 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 11.373209953308105 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 11.374008178710938 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 11.367918014526367 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 11.36426067352295 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 11.365376472473145 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 11.364357948303223 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 11.360784530639648 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 11.377557754516602 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 11.379057884216309 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 11.368156433105469 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 11.364771842956543 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 11.399645805358887 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 11.366986274719238 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 11.380050659179688 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 11.379220008850098 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 11.395848274230957 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 11.379345893859863 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 11.368217468261719 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 11.370192527770996 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 11.365897178649902 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 11.383872032165527 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 11.370857238769531 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 11.393929481506348 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 11.377248764038086 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 11.318794250488281 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 11.28624153137207 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 11.258989334106445 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 8.597808837890625 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 8.598052978515625 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 8.592071533203125 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 8.59305477142334 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 8.590449333190918 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 8.599859237670898 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 8.589568138122559 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 8.593244552612305 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 8.598782539367676 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 8.59886646270752 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 8.592616081237793 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 8.593147277832031 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 8.589019775390625 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 8.59089183807373 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 8.58103084564209 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 8.587532997131348 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 8.595832824707031 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 8.596173286437988 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 8.594398498535156 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 8.597038269042969 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 8.587000846862793 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 8.59517765045166 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 8.596869468688965 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 8.595001220703125 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 8.605178833007812 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 8.58203125 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 8.595943450927734 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 8.592460632324219 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 8.598711013793945 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 8.598311424255371 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 8.604592323303223 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 8.599326133728027 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 8.587484359741211 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 8.586885452270508 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 8.59148120880127 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 8.598372459411621 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 8.598960876464844 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 8.604971885681152 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 8.593976020812988 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 8.591090202331543 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 8.594829559326172 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 8.5955171585083 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 8.597284317016602 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 8.599096298217773 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 8.592257499694824 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 8.586142539978027 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 8.586615562438965 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 8.593001365661621 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 8.589214324951172 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 8.590858459472656 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 8.595794677734375 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 8.591263771057129 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 8.594550132751465 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 8.58612060546875 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 8.590489387512207 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 8.588607788085938 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 8.599702835083008 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 8.599637985229492 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 8.600960731506348 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 8.5936279296875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 8.59095287322998 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 8.592122077941895 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 8.591229438781738 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 8.596339225769043 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 8.59343147277832 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 8.602795600891113 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 8.604223251342773 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 8.59350872039795 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 8.591591835021973 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 8.593202590942383 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 8.59630012512207 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 8.602360725402832 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 8.595474243164062 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 8.595709800720215 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 8.594186782836914 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 8.598832130432129 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 8.59660816192627 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 8.599984169006348 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 8.596222877502441 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 8.59980583190918 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 8.595144271850586 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 8.59531307220459 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 8.596796989440918 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 8.595850944519043 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 8.596628189086914 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 8.597403526306152 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 8.598543167114258 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 8.595672607421875 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 8.592780113220215 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 8.600465774536133 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 8.593103408813477 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 8.598368644714355 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 8.596123695373535 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 8.596037864685059 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 8.591477394104004 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 8.593873977661133 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 8.601378440856934 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 8.591788291931152 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 8.594496726989746 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 8.591854095458984 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 8.596148490905762 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 8.590309143066406 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 8.595931053161621 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 8.596757888793945 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 8.59602165222168 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 8.597665786743164 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 8.597064971923828 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 8.594005584716797 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 8.598158836364746 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 8.59311580657959 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 8.599039077758789 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 8.594337463378906 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 8.59836196899414 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 8.599563598632812 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 8.597493171691895 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 8.594705581665039 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 8.592716217041016 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 8.5960054397583 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 8.59383487701416 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 8.598577499389648 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 8.600232124328613 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 8.597537994384766 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 8.597587585449219 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 8.595369338989258 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 8.59712028503418 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 8.594205856323242 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 8.599334716796875 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 8.59843635559082 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 8.597419738769531 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 8.597899436950684 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 8.593058586120605 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 8.596960067749023 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 8.591407775878906 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 8.599640846252441 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 8.598493576049805 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 8.59380054473877 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 8.599838256835938 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 8.592819213867188 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 8.597029685974121 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 8.596501350402832 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 8.599404335021973 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 8.599943161010742 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 8.599233627319336 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 8.599267959594727 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 8.595049858093262 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 8.599871635437012 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 8.5957670211792 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 8.599618911743164 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 8.59830093383789 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 8.597501754760742 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 8.599178314208984 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 8.595636367797852 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 8.598406791687012 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 8.593755722045898 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 8.599510192871094 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 8.59992504119873 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 8.59942626953125 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 8.600461959838867 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 8.593652725219727 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 8.597079277038574 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 8.59421443939209 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 8.599640846252441 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 8.599297523498535 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 8.599137306213379 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 8.598953247070312 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 8.596136093139648 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 8.594475746154785 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 8.595193862915039 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 8.600722312927246 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 8.601634979248047 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 8.600810050964355 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 8.599346160888672 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 8.594758987426758 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 8.595670700073242 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 8.593750953674316 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 8.599863052368164 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 8.597648620605469 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 8.600252151489258 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 8.598627090454102 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 8.595049858093262 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 8.593779563903809 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 8.594457626342773 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 8.59763240814209 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 8.598999977111816 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 8.600335121154785 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 8.59768009185791 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 8.594533920288086 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 8.596505165100098 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 8.596292495727539 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 8.599892616271973 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 8.597528457641602 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 8.598862648010254 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 8.59852409362793 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 8.59518814086914 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 8.5993013381958 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 8.595757484436035 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 8.598780632019043 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 8.599124908447266 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 8.600756645202637 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 8.599139213562012 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 8.596649169921875 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 8.598718643188477 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 8.596112251281738 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 8.601913452148438 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 8.598934173583984 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 8.59988784790039 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 8.598033905029297 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 8.596000671386719 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 8.57927131652832 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 8.597234725952148 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 8.600592613220215 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 8.599108695983887 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 8.60299015045166 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 8.599475860595703 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 8.598971366882324 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 8.595800399780273 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 8.598587989807129 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 8.600590705871582 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 8.596673011779785 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 8.602526664733887 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 8.60252857208252 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 8.574725151062012 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 8.58675479888916 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 8.554524421691895 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.837277889251709 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.836214542388916 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.834653377532959 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.834012508392334 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.835837364196777 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.835499286651611 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.834324359893799 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.836151123046875 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.838049411773682 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.837244510650635 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.836434841156006 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.833128929138184 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.832881927490234 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.832605838775635 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.8334574699401855 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.832785129547119 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.833269119262695 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.836244583129883 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.831515312194824 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.835611343383789 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.835990905761719 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.836042404174805 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.834421634674072 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.832755088806152 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.8353776931762695 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.831989765167236 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.835078239440918 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.831748962402344 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.835531234741211 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.8354339599609375 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.834109306335449 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.8355607986450195 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.831489562988281 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.833771705627441 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.833177089691162 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.837649822235107 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.836966037750244 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.836300849914551 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.836812496185303 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.834746360778809 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.835651874542236 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.834426403045654 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.835319995880127 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.836677551269531 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.8360595703125 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.833513259887695 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.8328118324279785 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.836398124694824 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.834989070892334 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.837219715118408 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.83495569229126 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.838012218475342 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.835457801818848 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.832639694213867 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.834014415740967 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.834107398986816 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.839770793914795 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.836799621582031 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.836124897003174 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.834132194519043 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.834257125854492 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.834567070007324 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.836083889007568 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.8364129066467285 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.8338704109191895 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.839615345001221 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.836932182312012 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.834951400756836 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.836427688598633 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.834622859954834 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.835848331451416 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.838118553161621 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.835777759552002 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.8363566398620605 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.835964202880859 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.836177349090576 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.835811138153076 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.836005687713623 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.836016654968262 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.836784362792969 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.835841178894043 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.8353962898254395 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.836023807525635 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.837029457092285 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.835930347442627 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.836324691772461 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.836704254150391 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.834599018096924 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.8344550132751465 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.836147308349609 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.836782455444336 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.835896968841553 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.836149215698242 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.834333419799805 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.833871841430664 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.834670066833496 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.837738990783691 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.836708068847656 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.836115837097168 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.833707809448242 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.835837364196777 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.835351467132568 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.835071563720703 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.838397026062012 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.83414888381958 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.8368611335754395 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.835990905761719 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.8359551429748535 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.834752082824707 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.835149765014648 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.8368401527404785 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.834599018096924 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.836528778076172 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.836031436920166 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.8372087478637695 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.836001873016357 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.833218574523926 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.836453437805176 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.832245349884033 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.837361812591553 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.837447643280029 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.836050510406494 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.835531234741211 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.83530330657959 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.8347554206848145 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.834499835968018 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.836946964263916 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.837053298950195 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.835824489593506 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.835681915283203 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.833207130432129 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.834996223449707 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.835026741027832 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.837042331695557 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.836808681488037 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.836740970611572 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.836771488189697 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.834537029266357 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.835299491882324 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.835809230804443 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.837578296661377 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.836580753326416 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.836535930633545 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.836946964263916 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.834822654724121 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.836379051208496 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.834419250488281 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.837069988250732 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.8370442390441895 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.837033271789551 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.835971832275391 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.8336687088012695 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.8357439041137695 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.834630489349365 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.836610317230225 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.836905479431152 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.836326599121094 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.83616828918457 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.8351240158081055 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.8369035720825195 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.83413028717041 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.837185859680176 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.837358474731445 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.8364763259887695 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.836121559143066 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.834957599639893 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.835489749908447 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.834989070892334 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.8379225730896 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.837162017822266 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.837306022644043 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.837369441986084 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.835508346557617 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.835818290710449 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.835183143615723 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.836537837982178 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.836442470550537 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.83662223815918 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.836221694946289 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.834753513336182 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.835424423217773 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.835134983062744 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.8365020751953125 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.8371710777282715 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.837535858154297 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.837535858154297 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.835310459136963 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.835412979125977 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.835855960845947 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.836109638214111 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.837103843688965 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.835966110229492 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.836289405822754 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.835206031799316 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.835633277893066 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.834432601928711 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.836935997009277 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.836434841156006 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.837181091308594 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.837120532989502 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.834465980529785 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.835792541503906 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.835870742797852 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.836734294891357 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.836558818817139 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.837937355041504 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.837421894073486 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.835514068603516 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.8358073234558105 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.834428310394287 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.8371992111206055 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.8363566398620605 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.838240146636963 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.837219715118408 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.834934711456299 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.836657524108887 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.835616588592529 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.837395191192627 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.83680534362793 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.838651180267334 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.836984634399414 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.828728199005127 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.824702739715576 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.820794105529785 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.513161659240723 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.51261568069458 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.512673377990723 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.5111846923828125 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.5122809410095215 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.513366222381592 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.512961387634277 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.512190818786621 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.513667106628418 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.5134124755859375 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.5101752281188965 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.5111846923828125 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.5124311447143555 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.512370586395264 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.510921478271484 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.511854648590088 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.513545513153076 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.512123107910156 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.513260841369629 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.512338161468506 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.512148380279541 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.511406898498535 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.512060642242432 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.512556552886963 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.5106730461120605 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.5127787590026855 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.513284206390381 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.512834072113037 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.51267147064209 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.512198448181152 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.512950897216797 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.512805461883545 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.509881019592285 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.512912750244141 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.512975692749023 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.512785911560059 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.512352466583252 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.513514995574951 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.512080669403076 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.510354042053223 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.511646747589111 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.51334810256958 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.513636589050293 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.513525485992432 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.5118632316589355 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.511097431182861 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.511048793792725 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.51108980178833 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.513273239135742 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.51328706741333 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.511163234710693 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.51058292388916 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.512445449829102 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.5112762451171875 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.512753963470459 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.511392593383789 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.5142621994018555 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.512722969055176 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.513660430908203 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.514659881591797 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.51156759262085 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.511500358581543 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.511571407318115 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.512943267822266 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.512252330780029 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.5146565437316895 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.512831211090088 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.5114030838012695 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.512509822845459 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.511980056762695 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.5135955810546875 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.513552188873291 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.512495994567871 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.51340389251709 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.512234210968018 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.512524127960205 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.512574672698975 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.5133376121521 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.512997150421143 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.51318359375 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.5125603675842285 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.512887954711914 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.512635231018066 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.5115647315979 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.513072490692139 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.513050556182861 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.51239538192749 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.511059284210205 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.512200355529785 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.5130438804626465 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.511865139007568 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.512895107269287 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.512800693511963 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.513131618499756 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.511455535888672 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.512327671051025 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.513027667999268 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.513345241546631 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.511894226074219 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.510855197906494 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.513133525848389 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.511302947998047 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.513212203979492 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.5129899978637695 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.511965751647949 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.512760639190674 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.511643409729004 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.512137413024902 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.511856555938721 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.511919021606445 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.512689590454102 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.5129241943359375 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.513290882110596 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.512993812561035 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.51235818862915 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.512922286987305 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.511840343475342 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.512676239013672 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.513754844665527 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.513529300689697 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.513273239135742 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.5123162269592285 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.512538433074951 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.511641502380371 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.513375282287598 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.512187957763672 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.513222694396973 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.5135416984558105 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.5132622718811035 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.5130295753479 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.5127716064453125 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.512348651885986 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.511435508728027 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.513405799865723 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.512208938598633 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.512948989868164 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.513269424438477 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.511868953704834 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.512531757354736 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.5118184089660645 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.513103008270264 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.513498783111572 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.513108253479004 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.5132246017456055 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.512144565582275 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.512026309967041 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.512485027313232 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.513341426849365 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.5136260986328125 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.512373924255371 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.513652801513672 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.511659622192383 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.512315273284912 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.512051105499268 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.513321399688721 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.5129241943359375 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.513011455535889 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.513017177581787 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.511087894439697 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.512696266174316 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.512139320373535 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.513312816619873 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.51353120803833 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.512918472290039 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.51334810256958 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.512028217315674 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.512406349182129 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.51115083694458 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.5135955810546875 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.513563632965088 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.51318359375 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.5135817527771 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.512453079223633 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.512520790100098 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.512266635894775 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.513488292694092 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.513362884521484 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.51309061050415 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.512607097625732 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.511965751647949 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.511800765991211 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.511774063110352 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.51301908493042 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.512837886810303 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.512805461883545 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.512225151062012 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.511994361877441 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.513047695159912 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.511983871459961 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.5132622718811035 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.51230001449585 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.513520240783691 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.512628555297852 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.512585163116455 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.5128397941589355 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.513171195983887 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.513379096984863 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.513238906860352 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.512546062469482 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.51318359375 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.5117363929748535 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.513740539550781 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.512755393981934 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.512991905212402 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.513681411743164 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.513472080230713 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.513428688049316 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.511700630187988 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.511733055114746 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.513296604156494 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.513387680053711 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.513396739959717 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.513457298278809 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.513084888458252 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.512631893157959 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.512370586395264 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.512619495391846 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.513405799865723 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.513072490692139 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.513982772827148 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.5135040283203125 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.508457660675049 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.506404876708984 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.504669189453125 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.349105358123779 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.348998546600342 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.348657608032227 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.348606586456299 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.348315238952637 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.348488807678223 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.34819221496582 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.348746299743652 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.3497138023376465 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.34853458404541 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.348019123077393 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.347789764404297 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.348057746887207 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.349111557006836 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.348143577575684 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.347986221313477 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.348141670227051 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.348902702331543 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.346934795379639 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.34902811050415 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.347555160522461 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.348958492279053 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.34832763671875 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.347682476043701 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.348515033721924 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.347317695617676 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.348386764526367 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.34850549697876 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.349215984344482 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.348427772521973 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.347492218017578 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.348297595977783 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.347870349884033 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.3483686447143555 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.348843097686768 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.349037170410156 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.349007606506348 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.348401069641113 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.348677158355713 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.348729610443115 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.349294662475586 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.348188400268555 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.348525047302246 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.349152565002441 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.3488898277282715 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.348543167114258 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.347622394561768 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.349116325378418 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.347233295440674 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.348350524902344 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.348527431488037 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.347989082336426 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.348415374755859 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.34763240814209 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.348501682281494 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.34812593460083 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.348637580871582 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.348945140838623 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.3488616943359375 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.347490310668945 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.3480143547058105 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.348774433135986 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.34886360168457 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.349031925201416 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.347808361053467 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.348097801208496 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.347846508026123 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.348308563232422 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.348904132843018 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.348912715911865 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.349771499633789 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.349147796630859 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.3488569259643555 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.347409725189209 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.348785877227783 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.348465442657471 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.348672866821289 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.348705768585205 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.348448276519775 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.348386764526367 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.348947048187256 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.3482279777526855 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.34935998916626 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.349040985107422 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.348435401916504 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.3486199378967285 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.349200248718262 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.348763942718506 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.348359107971191 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.348752021789551 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.348560810089111 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.349295616149902 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.348665714263916 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.34861946105957 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.349164962768555 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.348114967346191 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.349006175994873 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.34774923324585 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.348621368408203 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.34769344329834 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.347812652587891 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.348701000213623 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.348207950592041 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.348908424377441 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.348330974578857 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.349200248718262 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.348335266113281 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.34912109375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.349566459655762 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.347525596618652 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.34946870803833 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.347509860992432 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.348865509033203 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.348839282989502 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.348974704742432 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.348672866821289 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.348024368286133 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.349218368530273 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.346370697021484 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.3491716384887695 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.349269390106201 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.348747253417969 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.348944187164307 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.348488807678223 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.348266124725342 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.347610950469971 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.349203586578369 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.348564147949219 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.348825454711914 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.348113536834717 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.3477091789245605 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.348984718322754 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.349109172821045 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.348691463470459 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.348746299743652 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.348606586456299 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.348849773406982 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.348703384399414 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.348952293395996 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.348094463348389 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.348984718322754 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.34898042678833 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.348974704742432 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.349284648895264 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.348520755767822 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.3497819900512695 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.348166465759277 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.349273681640625 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.349025726318359 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.349084854125977 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.348677635192871 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.348645210266113 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.347969055175781 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.347885608673096 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.3489813804626465 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.349255561828613 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.349320411682129 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.349072456359863 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.34848165512085 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.348248481750488 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.34816837310791 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.349063396453857 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.348869323730469 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.349040985107422 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.348945140838623 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.348155498504639 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.348214149475098 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.348806381225586 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.349252700805664 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.348958492279053 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.34932279586792 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.348855972290039 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.347745418548584 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.348052024841309 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.347931861877441 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.348920822143555 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.348924160003662 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.349440097808838 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.349419116973877 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.348940849304199 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.348515510559082 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.3485212326049805 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.3489813804626465 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.349184036254883 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.349503040313721 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.349020481109619 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.348176956176758 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.348317623138428 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.34869909286499 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.349174499511719 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.348830223083496 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.348760604858398 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.349056720733643 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.348001003265381 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.3487324714660645 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.348695278167725 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.348966121673584 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.348723411560059 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.349458694458008 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.3485612869262695 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.3484086990356445 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.348546504974365 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.348423957824707 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.349245071411133 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.349086761474609 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.348988056182861 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.348770618438721 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.348517417907715 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.348731517791748 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.3474955558776855 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.349368095397949 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.348635196685791 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.349459171295166 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.349857330322266 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.34869909286499 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.348385334014893 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.348912715911865 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.349249362945557 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.34926700592041 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.3501739501953125 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.349207401275635 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.345584869384766 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.3456878662109375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.3424072265625 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.231305122375488 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.231243133544922 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.230782985687256 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.230456352233887 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.2309956550598145 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.23151969909668 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.230586528778076 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.230974197387695 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.231302738189697 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.2310967445373535 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.230555534362793 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.230952262878418 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.23143196105957 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.230432987213135 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.230630874633789 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.231547832489014 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.230907440185547 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.231179714202881 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.231357097625732 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.230424404144287 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.230627059936523 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.230669021606445 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.231324195861816 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.231151580810547 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.23089075088501 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.230734825134277 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.23164176940918 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.231616497039795 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.231032848358154 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.230863094329834 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.231847763061523 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.231514930725098 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.231203556060791 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.231055736541748 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.23005485534668 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.231025218963623 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.2310614585876465 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.231332778930664 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.230926990509033 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.230051040649414 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.230650424957275 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.231326103210449 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.231377124786377 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.231413841247559 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.231359004974365 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.231215000152588 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.2306742668151855 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.231349945068359 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.231532096862793 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.231412410736084 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.230489253997803 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.231984615325928 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.230673789978027 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.23034143447876 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.230419158935547 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.230432987213135 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.231305122375488 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.231396198272705 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.231278419494629 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.231464385986328 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.230872631072998 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.230029582977295 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.230785369873047 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.2310895919799805 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.230505466461182 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.231380939483643 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.23199462890625 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.2311272621154785 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.230792999267578 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.230129241943359 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.230911731719971 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.231540203094482 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.231377601623535 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.2319655418396 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.230907440185547 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.2304558753967285 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.231200218200684 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.231240749359131 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.230983257293701 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.231642246246338 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.230356693267822 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.231356143951416 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.231056213378906 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.231357097625732 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.231201648712158 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.2310709953308105 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.230929374694824 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.230267524719238 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.230993270874023 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.2308573722839355 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.230882167816162 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.231380462646484 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.231193542480469 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.230970859527588 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.230409622192383 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.23093843460083 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.231137275695801 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.231241226196289 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.231059551239014 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.229687213897705 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.231389999389648 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.23015832901001 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.230910301208496 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.230950355529785 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.231908798217773 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.2310380935668945 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.230284690856934 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.231332302093506 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.230702877044678 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.231137275695801 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.230822563171387 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.23063850402832 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.231051921844482 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.23117208480835 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.230563163757324 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.231016635894775 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.230430603027344 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.230780601501465 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.231510639190674 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.2311859130859375 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.230910301208496 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.230936527252197 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.230811595916748 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.230881214141846 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.231285095214844 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.23037576675415 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.230834484100342 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.231383323669434 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.231024265289307 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.231189727783203 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.2309346199035645 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.231051445007324 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.2298173904418945 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.231332778930664 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.230710983276367 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.230953693389893 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.231276035308838 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.230393409729004 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.230942726135254 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.230762004852295 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.231044292449951 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.231264114379883 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.231149196624756 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.231055736541748 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.230470180511475 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.230710983276367 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.2309489250183105 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.231112957000732 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.2314019203186035 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.23106050491333 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.231332302093506 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.230249404907227 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.231098651885986 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.231097221374512 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.2311577796936035 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.231036186218262 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.230983257293701 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.230825424194336 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.231053829193115 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.231044769287109 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.231168270111084 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.231233596801758 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.231088161468506 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.231139659881592 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.23115348815918 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.230447769165039 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.230926036834717 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.230709075927734 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.231558799743652 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.23145866394043 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.231086730957031 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.231522560119629 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.231419563293457 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.2311201095581055 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.231355667114258 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.23122501373291 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.231393337249756 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.231022834777832 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.231079578399658 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.230343341827393 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.2306227684021 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.230831623077393 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.2309794425964355 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.231173038482666 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.230711460113525 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.231256484985352 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.2310967445373535 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.230952739715576 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.2307281494140625 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.2311296463012695 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.2310051918029785 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.2311811447143555 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.230738162994385 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.230879306793213 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.230797290802002 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.2302141189575195 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.23129415512085 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.231047630310059 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.231017112731934 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.231252193450928 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.230777740478516 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.2318291664123535 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.230716705322266 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.231049537658691 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.231013298034668 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.231439590454102 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.230712413787842 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.230633735656738 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.230835437774658 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.231184005737305 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.230997562408447 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.231091499328613 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.231242656707764 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.230889797210693 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.2307658195495605 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.230627536773682 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.230912208557129 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.231451511383057 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.231168270111084 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.231124401092529 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.231187343597412 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.229653835296631 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.228086471557617 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.227556228637695 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.1670756340026855 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.167077541351318 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.166482925415039 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.166510105133057 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.166473388671875 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.167043209075928 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.166877269744873 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.166747093200684 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.16703462600708 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.167116641998291 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.166218280792236 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.166722297668457 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.166018009185791 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.166836261749268 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.166932582855225 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.1668877601623535 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.166880130767822 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.166190147399902 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.165665149688721 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.167118072509766 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.16711950302124 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.167108058929443 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.166270732879639 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.166565895080566 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.166510581970215 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.165586471557617 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.166639804840088 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.165777206420898 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.167055130004883 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.166831016540527 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.166677474975586 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.16681432723999 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.166489601135254 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.166824817657471 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.167451858520508 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.167075157165527 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.16724157333374 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.166859149932861 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.16680383682251 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.166874885559082 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.1669602394104 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.166453838348389 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.167015075683594 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.16743803024292 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.16640043258667 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.165816307067871 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.1663947105407715 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.167409896850586 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.166446685791016 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.166971683502197 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.166647434234619 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.166383266448975 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.167677402496338 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.1667304039001465 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.167272090911865 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.165804386138916 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.166803359985352 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.1668572425842285 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.16705846786499 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.166172027587891 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.16665506362915 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.1671223640441895 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.166554927825928 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.167471408843994 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.166079521179199 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.1669721603393555 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.166313648223877 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.166508197784424 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.166845798492432 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.166477680206299 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.167202472686768 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.167214393615723 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.1665120124816895 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.166686058044434 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.166950225830078 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.167572021484375 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.166472434997559 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.167119979858398 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.166769981384277 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.16677713394165 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.167628288269043 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.166843414306641 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.1670122146606445 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.166385173797607 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.166908264160156 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.166764736175537 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.166801929473877 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.16705322265625 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.166453838348389 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.166936874389648 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.166955947875977 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.166982650756836 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.166708469390869 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.166819095611572 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.1665472984313965 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.166648864746094 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.166975021362305 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.166632175445557 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.166874408721924 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.166378498077393 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.166443347930908 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.1667938232421875 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.1667375564575195 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.167116641998291 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.166187286376953 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.167230129241943 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.166417121887207 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.1666364669799805 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.166772842407227 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.166645050048828 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.1669135093688965 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.166627883911133 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.166938304901123 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.166843414306641 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.166697978973389 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.166558265686035 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.166444778442383 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.167336463928223 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.165580749511719 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.167168140411377 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.166784763336182 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.166501045227051 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.166749477386475 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.166286945343018 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.166374683380127 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.166357517242432 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.166918754577637 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.1667799949646 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.166805744171143 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.166475772857666 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.166627883911133 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.166595458984375 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.166696071624756 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.167073726654053 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.166415214538574 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.166646957397461 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.166938304901123 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.166426658630371 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.1673455238342285 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.16665506362915 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.1670637130737305 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.167266368865967 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.166731834411621 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.166886329650879 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.166549205780029 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.167111873626709 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.166107654571533 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.16725492477417 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.166863441467285 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.167104721069336 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.166479110717773 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.166940689086914 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.166525363922119 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.166565895080566 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.166836261749268 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.16712760925293 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.166840553283691 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.16696310043335 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.16658878326416 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.1671833992004395 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.166407585144043 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.166825294494629 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.1669697761535645 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.167006492614746 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.167112827301025 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.167008876800537 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.166055202484131 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.166828155517578 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.166959762573242 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.167143821716309 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.167054176330566 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.167110443115234 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.166277885437012 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.166382312774658 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.166240215301514 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.167092800140381 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.166874408721924 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.16717004776001 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.166975498199463 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.1665239334106445 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.166716575622559 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.167118072509766 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.166955947875977 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.166975021362305 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.167444229125977 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.166790008544922 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.166225910186768 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.166610240936279 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.16623067855835 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.167023181915283 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.166938781738281 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.166840553283691 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.167097568511963 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.1665802001953125 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.166973114013672 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.167262554168701 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.167046546936035 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.16680383682251 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.16702127456665 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.166765213012695 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.166347980499268 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.166127681732178 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.167111873626709 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.167100429534912 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.1670918464660645 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.166886329650879 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.16727876663208 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.166932582855225 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.1671953201293945 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.166347980499268 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.16701602935791 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.166863441467285 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.167264938354492 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.167359352111816 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.166518211364746 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.1672468185424805 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.167033672332764 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.166965961456299 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.167123794555664 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.167694568634033 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.1669697761535645 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.165483474731445 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.165350914001465 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.163064002990723 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.102043628692627 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.102055549621582 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.102056503295898 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.10192346572876 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.102001667022705 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.102075576782227 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.101577281951904 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.1023640632629395 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.102118015289307 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.10202693939209 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.101953506469727 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.102345943450928 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.102297306060791 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.101475715637207 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.101492881774902 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.101742744445801 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.1020612716674805 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.103018760681152 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.102550983428955 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.101711273193359 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.101040840148926 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.101881504058838 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.1022868156433105 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.1014251708984375 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.101716995239258 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.1024274826049805 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.1026387214660645 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.102385520935059 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.102137565612793 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.101955413818359 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.102227687835693 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.101766109466553 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.102075576782227 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.102334976196289 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.1011199951171875 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.102029800415039 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.102337837219238 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.10210657119751 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.102035999298096 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.10156774520874 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.1022820472717285 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.102056503295898 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.101751804351807 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.102110385894775 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.102485179901123 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.10270881652832 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.101874828338623 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.101753234863281 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.102033615112305 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.102246284484863 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.101475715637207 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.101795673370361 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.101335048675537 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.1015520095825195 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.101592540740967 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.102215766906738 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.102473258972168 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.102231025695801 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.102205753326416 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.102241039276123 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.101919651031494 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.102010726928711 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.102048873901367 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.10205078125 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.101192474365234 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.102075576782227 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.1024088859558105 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.1022443771362305 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.102112293243408 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.101648330688477 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.1019744873046875 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.102248668670654 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.102403163909912 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.1022467613220215 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.102100849151611 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.101172924041748 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.1018805503845215 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.102094650268555 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.102227687835693 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.102269649505615 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.100893497467041 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.101828098297119 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.101785182952881 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.102273464202881 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.102163791656494 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.102112293243408 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.102126121520996 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.101513862609863 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.102169990539551 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.101889133453369 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.102219104766846 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.101894378662109 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.102065086364746 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.101651191711426 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.101280689239502 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.102139472961426 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.1019511222839355 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.101823806762695 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.101961135864258 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.101353645324707 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.101902008056641 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.101451873779297 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.1020026206970215 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.101641654968262 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.102087020874023 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.101875305175781 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.1019463539123535 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.1018500328063965 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.101994037628174 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.102202415466309 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.102303981781006 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.102282524108887 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.102014541625977 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.1020660400390625 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.10206413269043 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.101678848266602 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.10137939453125 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.101630687713623 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.102574825286865 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.102242946624756 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.1019721031188965 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.1020050048828125 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.101560115814209 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.101799011230469 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.10227108001709 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.101616859436035 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.102297306060791 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.10222864151001 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.10202693939209 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.102185249328613 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.101616859436035 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.101884365081787 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.101611137390137 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.102083683013916 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.101726531982422 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.101886749267578 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.102127552032471 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.101536273956299 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.101653099060059 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.101969242095947 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.102148532867432 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.1020050048828125 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.102456569671631 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.102106094360352 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.101814270019531 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.102104663848877 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.1019368171691895 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.101951599121094 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.10238790512085 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.102085590362549 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.102309226989746 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.1015448570251465 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.102012634277344 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.10172176361084 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.1019721031188965 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.1023359298706055 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.102123737335205 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.10196590423584 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.101649284362793 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.101936340332031 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.102182865142822 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.102267265319824 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.102281093597412 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.102225303649902 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.10186243057251 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.101739883422852 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.102254390716553 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.1014604568481445 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.102395534515381 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.1022772789001465 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.102282524108887 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.1021928787231445 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.101942539215088 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.101826190948486 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.102313995361328 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.102252006530762 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.102383613586426 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.102126121520996 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.101846694946289 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.101456642150879 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.101865291595459 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.101384162902832 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.1021623611450195 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.102158069610596 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.102006912231445 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.101912975311279 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.101895809173584 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.102232933044434 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.102006912231445 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.102298736572266 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.102091312408447 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.102104187011719 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.101900577545166 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.102149963378906 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.101412296295166 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.101471424102783 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.102121829986572 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.101968765258789 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.101958274841309 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.102278709411621 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.101814270019531 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.10270881652832 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.101851940155029 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.102148532867432 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.101959228515625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.102565765380859 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.10183572769165 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.101572513580322 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.101951599121094 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.102169513702393 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.102046966552734 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.102073669433594 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.102302074432373 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.102075576782227 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.1019721031188965 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.101607322692871 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.101987361907959 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.102264881134033 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.101897716522217 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.101887226104736 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.102479934692383 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.100865364074707 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.099358081817627 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.100358486175537 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.055805683135986 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.055902481079102 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.05539608001709 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.054920673370361 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.055208206176758 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.055775165557861 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.055538654327393 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.055654048919678 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.055642604827881 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.055302619934082 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.055319309234619 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.055445194244385 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.055363178253174 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.05544376373291 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.0556793212890625 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.055312633514404 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.05564546585083 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.05467414855957 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.055346488952637 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.055977821350098 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.0555338859558105 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.055515766143799 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.055302619934082 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.055177211761475 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.055350303649902 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.055146217346191 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.0549774169921875 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.0545334815979 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.055339336395264 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.05544376373291 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.055205821990967 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.055644989013672 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.055431842803955 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.055152893066406 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.055941581726074 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.055622100830078 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.055415153503418 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.055148601531982 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.055572986602783 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.055505275726318 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.055110931396484 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.055179119110107 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.055775165557861 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.055690288543701 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.054978370666504 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.054718971252441 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.055396556854248 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.055384159088135 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.055196285247803 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.055325984954834 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.055329322814941 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.055420398712158 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.0555548667907715 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.055493354797363 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.055706024169922 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.055211544036865 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.055383205413818 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.055490493774414 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.055425643920898 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.055116176605225 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.055275917053223 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.055375576019287 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.055789470672607 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.055596828460693 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.054478168487549 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.0558295249938965 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.055278778076172 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.055333614349365 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.055490970611572 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.055747032165527 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.0556254386901855 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.0555291175842285 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.055234432220459 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.055374622344971 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.055263996124268 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.056067943572998 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.055290699005127 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.055603504180908 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.0553059577941895 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.055549621582031 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.0554656982421875 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.055549144744873 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.055830955505371 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.055342197418213 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.055544376373291 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.055522918701172 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.055146217346191 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.0551958084106445 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.055310249328613 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.055346488952637 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.05544900894165 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.055787563323975 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.055458068847656 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.05547571182251 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.055437088012695 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.055289268493652 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.0553998947143555 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.055469989776611 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.055727958679199 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.055065155029297 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.05530309677124 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.055521011352539 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.055472373962402 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.055646896362305 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.05546236038208 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.055747032165527 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.0554399490356445 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.055659294128418 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.055502414703369 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.0553178787231445 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.055413246154785 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.054725646972656 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.05561637878418 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.055577754974365 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.054944038391113 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.055410385131836 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.055059909820557 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.055785179138184 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.054453372955322 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.055805683135986 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.055363655090332 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.05532693862915 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.055869102478027 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.055251121520996 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.054939270019531 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.055329322814941 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.055511474609375 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.055588245391846 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.05559778213501 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.0551981925964355 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.055211067199707 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.055384159088135 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.055129528045654 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.055819034576416 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.054951190948486 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.055706024169922 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.055627346038818 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.055325508117676 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.055548191070557 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.055455207824707 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.055587291717529 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.055707931518555 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.055556774139404 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.055625915527344 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.055504322052002 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.055583477020264 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.055618762969971 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.055825710296631 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.055638790130615 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.055821895599365 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.055408477783203 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.055321216583252 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.055190086364746 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.055345058441162 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.055726051330566 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.055642604827881 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.055426120758057 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.055692672729492 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.05545711517334 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.0555291175842285 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.054787635803223 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.0555901527404785 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.055582046508789 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.055680274963379 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.055591106414795 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.055134296417236 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.0552473068237305 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.055331230163574 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.055697441101074 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.0556440353393555 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.055775165557861 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.0558295249938965 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.055618762969971 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.055507659912109 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.0547027587890625 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.055726528167725 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.05553674697876 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.0556488037109375 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.055832386016846 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.055769920349121 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.055434226989746 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.055913925170898 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.055821895599365 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.055477619171143 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.055902481079102 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.055773735046387 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.055186748504639 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.055234909057617 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.055479049682617 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.0556640625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.055568218231201 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.055638790130615 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.05605936050415 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.055233955383301 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.055811882019043 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.0556960105896 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.0555596351623535 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.055616855621338 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.055775165557861 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.0555100440979 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.055205345153809 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.055152893066406 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.05559778213501 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.055532932281494 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.05589485168457 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.055294036865234 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.0557146072387695 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.055536270141602 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.055526256561279 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.05524206161499 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.055946350097656 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.055373191833496 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.055810451507568 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.055604934692383 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.05588960647583 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.055452346801758 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.055365085601807 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.055790424346924 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.055927753448486 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.056197643280029 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.055755615234375 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.0543036460876465 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.054818153381348 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.053571701049805 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.011786460876465 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.0116190910339355 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.011609077453613 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.011497497558594 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.011344909667969 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.011737823486328 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.011728286743164 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.011603355407715 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.011682987213135 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.011577606201172 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.011270999908447 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.011564254760742 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.01149320602417 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.011186599731445 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.010980606079102 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.011658191680908 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.011318683624268 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.011856555938721 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.011181354522705 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.011288166046143 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.0116143226623535 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.011526107788086 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.011597156524658 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.011082649230957 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.011173248291016 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.011507511138916 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.012094974517822 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.012552738189697 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.011871814727783 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.011406421661377 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.012229919433594 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.01154088973999 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.011526107788086 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.011799335479736 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.0108537673950195 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.011416912078857 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.011547088623047 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.011910438537598 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.011457920074463 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.011120319366455 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.011648654937744 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.01122522354126 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.011380672454834 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.011673450469971 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.011512756347656 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.012056827545166 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.01116943359375 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.011726379394531 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.011661529541016 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.011580944061279 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.010778427124023 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.0111613273620605 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.011728286743164 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.011378765106201 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.011289596557617 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.0113043785095215 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.011753082275391 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.011533260345459 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.011551856994629 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.011401176452637 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.011602878570557 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.011414051055908 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.010955333709717 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.011611461639404 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.011082649230957 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.011538982391357 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.011358261108398 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.0114874839782715 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.011069297790527 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.011565685272217 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.011546611785889 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.011380672454834 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.011739730834961 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.011274814605713 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.011080265045166 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.011063098907471 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.011607646942139 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.011574745178223 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.011327266693115 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.011490345001221 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.011256694793701 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.011722087860107 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.011471271514893 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.011837482452393 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.011478900909424 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.011547088623047 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.011351108551025 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.011078834533691 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.0115885734558105 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.011202812194824 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.0113301277160645 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.011360168457031 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.011343002319336 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.011427879333496 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.011305809020996 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.011674880981445 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.011246204376221 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.011225700378418 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.01153564453125 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.01100492477417 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.011551856994629 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.0112128257751465 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.011563301086426 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.011744976043701 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.011341094970703 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.011591911315918 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.011279106140137 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.011454105377197 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.011106491088867 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.011666297912598 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.011203289031982 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.01193380355835 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.011374473571777 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.011656284332275 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.011873722076416 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.011320114135742 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.011025905609131 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.011091232299805 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.011373519897461 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.011505603790283 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.011198043823242 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.011352062225342 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.011291980743408 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.011499404907227 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.01179838180542 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.010815620422363 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.011526107788086 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.01127290725708 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.011360168457031 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.011495590209961 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.011520862579346 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.0110979080200195 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.011141300201416 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.011358261108398 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.0107421875 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.011157512664795 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.011528015136719 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.011253833770752 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.011085510253906 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.011074542999268 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.011622428894043 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.011648178100586 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.011403560638428 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.011280059814453 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.011189937591553 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.011457920074463 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.01110315322876 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.011293411254883 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.011632919311523 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.011410713195801 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.0117363929748535 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.0113525390625 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.011671543121338 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.011308670043945 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.011526107788086 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.011673450469971 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.011743068695068 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.011392593383789 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.011292457580566 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.011499881744385 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.011254787445068 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.011529445648193 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.011537551879883 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.0114617347717285 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.011586666107178 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.011180400848389 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.011337757110596 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.01141881942749 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.0116190910339355 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.011641025543213 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.011601448059082 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.011565685272217 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.011059761047363 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.01143741607666 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.011694431304932 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.011526107788086 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.011562347412109 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.0115509033203125 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.01143217086792 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.01145076751709 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.011473655700684 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.010941505432129 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.011422157287598 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.0115790367126465 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.0114922523498535 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.011216640472412 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.0111799240112305 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.011465549468994 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.0113654136657715 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.011656284332275 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.011444091796875 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.0113959312438965 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.011200904846191 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.011064529418945 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.011112689971924 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.011246681213379 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.011486053466797 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.0113677978515625 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.0114216804504395 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.011388778686523 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.0112385749816895 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.011910438537598 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.010953903198242 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.011320114135742 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.011429309844971 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.011688232421875 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.011667728424072 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.011043071746826 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.011017799377441 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.011480331420898 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.011435031890869 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.0114240646362305 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.01156759262085 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.011301040649414 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.011284351348877 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.0116376876831055 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.011344909667969 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.011541366577148 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.011407375335693 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.011390209197998 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.0115461349487305 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.011318683624268 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.0096025466918945 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.009438991546631 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.979372501373291 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.979671001434326 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.979470252990723 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.9788713455200195 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.979344844818115 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.979386329650879 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.978827953338623 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.979533672332764 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.979511260986328 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.979249954223633 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.979383945465088 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.9794464111328125 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.979742527008057 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.97996711730957 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.9796013832092285 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.979661464691162 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.979353904724121 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.978926658630371 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.979310035705566 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.97960901260376 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.979271411895752 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.97946834564209 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.979097843170166 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.979152202606201 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.979803085327148 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.979077339172363 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.978804588317871 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.978210926055908 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.979213237762451 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.979472637176514 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.978539943695068 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.979611873626709 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.979398250579834 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.9792094230651855 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.979714393615723 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.979506015777588 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.979254722595215 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.979483604431152 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.979661464691162 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.979559898376465 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.9790472984313965 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.979494571685791 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.979491233825684 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.979403972625732 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.979483127593994 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.978681564331055 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.979311943054199 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.97915506362915 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.97953987121582 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.979443550109863 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.97950553894043 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.9793782234191895 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.979196071624756 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.979472637176514 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.979358196258545 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.979695796966553 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.979397296905518 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.979156970977783 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.979488849639893 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.979223728179932 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.979293346405029 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.979223728179932 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.979470252990723 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.979186534881592 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.979028701782227 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.979389667510986 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.979314804077148 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.97939395904541 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.979692459106445 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.97939395904541 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.979699611663818 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.979438781738281 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.979429721832275 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.979493141174316 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.97966194152832 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.9796462059021 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.979147434234619 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.9795122146606445 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.979246616363525 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.97957181930542 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.979411602020264 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.979314804077148 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.97967529296875 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.978989601135254 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.979547023773193 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.979350566864014 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.978883266448975 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.979375839233398 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.979447364807129 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.979394912719727 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.979761123657227 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.979611396789551 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.979274749755859 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.979589462280273 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.979175090789795 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.9792866706848145 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.97938346862793 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.979477882385254 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.979233264923096 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.9791998863220215 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.979235649108887 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.979332447052002 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.979536056518555 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.979044437408447 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.97907018661499 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.979554653167725 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.979428291320801 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.979336738586426 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.9794745445251465 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.979148864746094 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.979631423950195 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.978733539581299 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.979323863983154 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.979273319244385 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.978865623474121 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.979246139526367 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.9791460037231445 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.979772567749023 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.978929042816162 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.979341983795166 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.979158401489258 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.979050636291504 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.979581832885742 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.978979110717773 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.979001998901367 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.979592800140381 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.979342937469482 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.979243278503418 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.979499816894531 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.979137897491455 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.97922945022583 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.979158401489258 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.979380130767822 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.979528903961182 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.978546142578125 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.979291915893555 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.979400157928467 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.979405879974365 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.979520797729492 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.979396820068359 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.979327201843262 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.979556560516357 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.979193687438965 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.9795355796813965 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.979293346405029 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.979560852050781 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.979440689086914 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.979344367980957 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.979507923126221 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.979389190673828 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.979231834411621 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.979261875152588 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.979010105133057 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.97908878326416 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.979479789733887 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.979366779327393 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.979249954223633 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.979504585266113 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.979465484619141 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.979246616363525 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.9789838790893555 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.979574203491211 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.979332447052002 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.979517936706543 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.979682445526123 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.979556560516357 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.979215145111084 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.979139804840088 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.979609489440918 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.9793267250061035 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.9795026779174805 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.979515075683594 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.979187965393066 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.979628086090088 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.978904724121094 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.979491233825684 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.979440689086914 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.979604244232178 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.979344844818115 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.979172706604004 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.979118824005127 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.979554653167725 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.979517936706543 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.979305744171143 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.97958517074585 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.979287624359131 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.979133605957031 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.979393005371094 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.97920560836792 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.979401111602783 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.979329586029053 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.979436874389648 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.979382038116455 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.9793925285339355 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.979306697845459 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.979251861572266 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.979344367980957 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.979318141937256 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.979568004608154 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.979576110839844 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.979185104370117 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.979413986206055 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.979464530944824 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.979365348815918 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.9795122146606445 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.9793500900268555 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.979356288909912 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.9795308113098145 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.979398250579834 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.979220390319824 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.979582786560059 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.9794230461120605 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.979544639587402 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.9796624183654785 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.97976016998291 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.979100227355957 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.979276180267334 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.979362487792969 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.97966194152832 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.979741096496582 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.979310035705566 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.978361129760742 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.97890567779541 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.978408336639404 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.952268123626709 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.952213287353516 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.9525675773620605 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.9521164894104 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.952381610870361 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.952338695526123 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.952295780181885 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.952470302581787 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.952394962310791 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.951811790466309 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.95188045501709 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.952483654022217 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.951958656311035 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.951382637023926 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.952046871185303 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.95226526260376 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.952523708343506 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.952690601348877 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.9524827003479 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.952003479003906 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.952183246612549 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.952294826507568 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.952385425567627 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.952093601226807 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.952171802520752 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.952206134796143 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.952854156494141 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.9527974128723145 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.952458381652832 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.952174186706543 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.9527082443237305 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.951969623565674 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.952164649963379 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.952415943145752 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.952348709106445 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.95220422744751 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.952489852905273 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.951938152313232 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.952115058898926 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.9516682624816895 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.952337265014648 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.952200412750244 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.95228910446167 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.952324390411377 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.951807022094727 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.952436447143555 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.9523468017578125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.952402114868164 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.952058792114258 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.952163219451904 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.951775550842285 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.951715469360352 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.952023983001709 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.952404975891113 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.952372074127197 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.952061653137207 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.952334880828857 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.952464580535889 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.952376842498779 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.9524641036987305 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.952149391174316 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.9522600173950195 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.952101707458496 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.952287673950195 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.952035427093506 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.952324390411377 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.952136516571045 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.952294826507568 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.952091217041016 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.951780319213867 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.952464580535889 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.952301502227783 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.952095985412598 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.951964855194092 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.952077865600586 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.951908111572266 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.952080249786377 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.952480792999268 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.95213508605957 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.951985836029053 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.952090263366699 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.9524102210998535 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.952280521392822 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.952195167541504 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.952346324920654 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.952299118041992 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.95220422744751 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.952070713043213 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.952132701873779 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.951939582824707 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.9521894454956055 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.9522809982299805 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.9522881507873535 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.951977729797363 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.952101707458496 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.952103137969971 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.952635288238525 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.951967716217041 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.952338695526123 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.951846122741699 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.9523468017578125 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.9522624015808105 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.9522247314453125 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.952445030212402 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.952335357666016 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.952254295349121 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.9518914222717285 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.952110290527344 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.95219087600708 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.952451705932617 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.95187520980835 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.952856063842773 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.95214319229126 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.952489852905273 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.952089786529541 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.9520039558410645 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.95218563079834 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.951988697052002 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.9521565437316895 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.952280521392822 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.952249050140381 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.952043533325195 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.951990604400635 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.952183246612549 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.952592372894287 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.951357841491699 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.952146530151367 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.952382564544678 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.952211380004883 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.952367782592773 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.952103137969971 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.951925277709961 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.952013969421387 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.952328205108643 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.951961994171143 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.952099800109863 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.952372074127197 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.9521331787109375 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.952103137969971 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.952101707458496 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.952271461486816 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.9522809982299805 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.952449798583984 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.9522600173950195 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.952043533325195 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.951996326446533 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.9516777992248535 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.952352046966553 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.952385902404785 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.952363014221191 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.952503204345703 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.951963424682617 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.952331066131592 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.952059745788574 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.952215194702148 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.952149868011475 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.95238733291626 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.9522175788879395 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.952200889587402 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.952198505401611 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.952418327331543 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.952307224273682 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.952200889587402 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.95228910446167 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.952192306518555 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.951810359954834 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.951929092407227 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.95236349105835 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.952411651611328 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.9524383544921875 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.95245361328125 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.952396869659424 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.952065467834473 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.951988697052002 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.952589511871338 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.95242166519165 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.952406883239746 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.952192783355713 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.952110290527344 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.951969623565674 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.9523515701293945 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.9520487785339355 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.952484607696533 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.952269554138184 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.9520392417907715 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.952179908752441 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.952279090881348 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.952056884765625 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.952088356018066 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.95235538482666 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.952307224273682 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.952115058898926 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.951972484588623 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.952033996582031 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.9518351554870605 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.952298164367676 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.952460765838623 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.952279090881348 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.952099800109863 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.951998233795166 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.951975345611572 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.952199935913086 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.952183246612549 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.952436447143555 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.952443599700928 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.952312469482422 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.952343940734863 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.952007293701172 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.952192306518555 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.952183246612549 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.9521589279174805 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.952247619628906 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.952099800109863 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.952062129974365 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.952091217041016 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.952054023742676 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.952073574066162 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.9522247314453125 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.952115058898926 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.952073574066162 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.95232629776001 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.951535224914551 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.9506731033325195 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.950507640838623 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.929144382476807 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.929317951202393 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.928995132446289 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.929056167602539 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.928905487060547 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.929407119750977 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.9292378425598145 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.929161548614502 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.929314136505127 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.92929744720459 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.929308891296387 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.92859411239624 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.929251670837402 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.9292192459106445 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.929402828216553 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.929551601409912 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.929041385650635 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.928858757019043 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.929431438446045 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.929997444152832 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.929115295410156 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.929383754730225 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.928980350494385 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.928955554962158 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.929349422454834 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.929417133331299 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.928493022918701 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.928234577178955 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.928955554962158 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.929269313812256 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.928290367126465 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.929201126098633 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.928948879241943 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.928752899169922 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.92909049987793 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.929112434387207 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.929267406463623 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.929194927215576 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.929278373718262 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.929452419281006 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.928376197814941 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.928870677947998 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.9290361404418945 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.9291534423828125 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.928894996643066 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.928924083709717 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.929086685180664 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.9291205406188965 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.929363250732422 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.929221153259277 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.928803443908691 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.92887020111084 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.929144859313965 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.928995132446289 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.9291181564331055 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.929235458374023 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.9291863441467285 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.928842544555664 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.928725242614746 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.928530693054199 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.92918586730957 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.9289727210998535 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.9295268058776855 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.92947244644165 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.92879056930542 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.929110527038574 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.928987503051758 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.929157257080078 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.929174900054932 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.92901611328125 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.929321765899658 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.929131507873535 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.928730487823486 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.929184913635254 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.929198265075684 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.92943000793457 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.92909049987793 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.929424285888672 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.929127216339111 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.92918586730957 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.929348468780518 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.929306983947754 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.929394721984863 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.929099082946777 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.929258823394775 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.92922306060791 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.929118633270264 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.928915500640869 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.929281234741211 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.929405689239502 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.929389476776123 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.9292755126953125 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.929200649261475 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.929435729980469 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.929018020629883 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.929349899291992 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.92914342880249 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.929568290710449 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.929352283477783 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.929314136505127 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.929105281829834 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.929492950439453 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.929203987121582 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.929079055786133 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.929198265075684 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.929314136505127 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.92931604385376 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.929261684417725 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.929152011871338 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.929196357727051 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.9291558265686035 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.928519248962402 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.92941951751709 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.929229259490967 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.92848539352417 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.929450035095215 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.929213523864746 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.929625511169434 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.929211616516113 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.929263591766357 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.929243087768555 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.928810119628906 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.929252624511719 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.92906379699707 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.928866863250732 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.929127216339111 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.92947244644165 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.929221153259277 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.929415225982666 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.929110527038574 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.929211616516113 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.929462432861328 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.929228782653809 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.9294209480285645 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.9291300773620605 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.929256439208984 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.92935848236084 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.928892612457275 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.929150104522705 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.929437160491943 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.929416656494141 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.929346084594727 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.929228782653809 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.929332733154297 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.929293632507324 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.929435729980469 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.929280757904053 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.929405689239502 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.929366588592529 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.92952299118042 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.929241180419922 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.929187774658203 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.929032325744629 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.929424285888672 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.929314136505127 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.929335117340088 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.929235935211182 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.929250240325928 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.929609298706055 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.929013729095459 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.9289093017578125 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.929329872131348 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.929369926452637 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.929252624511719 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.929422855377197 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.929327011108398 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.92921781539917 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.928919315338135 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.929573059082031 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.929383277893066 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.929314136505127 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.9294915199279785 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.92923641204834 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.928934574127197 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.928952693939209 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.929373741149902 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.929200649261475 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.929214954376221 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.929477214813232 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.929366588592529 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.929112434387207 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.929344654083252 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.929286003112793 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.929254531860352 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.929396152496338 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.92943811416626 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.928884029388428 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.929208755493164 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.929207801818848 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.929346561431885 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.9293718338012695 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.929437637329102 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.929508686065674 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.929377555847168 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.929414749145508 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.929027080535889 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.929328918457031 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.929170608520508 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.929458141326904 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.929463863372803 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.929394721984863 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.9291582107543945 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.9292378425598145 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.929340362548828 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.929207801818848 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.929275035858154 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.929244041442871 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.929267406463623 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.92917013168335 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.928857803344727 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.9293317794799805 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.929200172424316 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.929579734802246 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.929306507110596 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.929420471191406 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.929253101348877 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.929107189178467 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.929324150085449 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.929483890533447 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.929576396942139 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.929513931274414 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.928544521331787 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.929280757904053 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.929066181182861 + } + ] + } + ] +} \ No newline at end of file