diff --git "a/provenance/folding.json" "b/provenance/folding.json" new file mode 100644--- /dev/null +++ "b/provenance/folding.json" @@ -0,0 +1,44024 @@ +{ + "created_at": "2026-08-27T11:49:54.868081+00:00", + "fold": { + "compute_dtype": "float32", + "device": "cuda", + "equations": { + "down_proj": "P_out @ W_down", + "gate_proj": "W_gate @ P_in", + "router_proj": "R_glm[selected expert rows] @ P_in", + "up_proj": "W_up @ P_in" + }, + "expert_map": [ + { + "expert_slot": 0, + "source_expert_id": 149, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 1, + "source_expert_id": 171, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 2, + "source_expert_id": 186, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 3, + "source_expert_id": 98, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 4, + "source_expert_id": 286, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 5, + "source_expert_id": 189, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 6, + "source_expert_id": 12, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 7, + "source_expert_id": 10, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 8, + "source_expert_id": 202, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 9, + "source_expert_id": 177, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 10, + "source_expert_id": 66, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 11, + "source_expert_id": 24, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 12, + "source_expert_id": 118, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 13, + "source_expert_id": 107, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 14, + "source_expert_id": 220, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 15, + "source_expert_id": 47, + "source_layer": 3, + "target_layer": 0 + }, + { + "expert_slot": 0, + "source_expert_id": 241, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 1, + "source_expert_id": 243, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 2, + "source_expert_id": 173, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 3, + "source_expert_id": 268, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 4, + "source_expert_id": 39, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 5, + "source_expert_id": 45, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 6, + "source_expert_id": 216, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 7, + "source_expert_id": 197, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 8, + "source_expert_id": 205, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 9, + "source_expert_id": 223, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 10, + "source_expert_id": 89, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 11, + "source_expert_id": 178, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 12, + "source_expert_id": 248, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 13, + "source_expert_id": 93, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 14, + "source_expert_id": 142, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 15, + "source_expert_id": 74, + "source_layer": 4, + "target_layer": 1 + }, + { + "expert_slot": 0, + "source_expert_id": 160, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 1, + "source_expert_id": 276, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 2, + "source_expert_id": 104, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 3, + "source_expert_id": 234, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 4, + "source_expert_id": 16, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 5, + "source_expert_id": 83, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 6, + "source_expert_id": 209, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 7, + "source_expert_id": 260, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 8, + "source_expert_id": 270, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 9, + "source_expert_id": 282, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 10, + "source_expert_id": 55, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 11, + "source_expert_id": 174, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 12, + "source_expert_id": 236, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 13, + "source_expert_id": 256, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 14, + "source_expert_id": 137, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 15, + "source_expert_id": 206, + "source_layer": 6, + "target_layer": 2 + }, + { + "expert_slot": 0, + "source_expert_id": 81, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 1, + "source_expert_id": 57, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 2, + "source_expert_id": 131, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 3, + "source_expert_id": 233, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 4, + "source_expert_id": 68, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 5, + "source_expert_id": 129, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 6, + "source_expert_id": 94, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 7, + "source_expert_id": 145, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 8, + "source_expert_id": 208, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 9, + "source_expert_id": 262, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 10, + "source_expert_id": 184, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 11, + "source_expert_id": 48, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 12, + "source_expert_id": 211, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 13, + "source_expert_id": 51, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 14, + "source_expert_id": 44, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 15, + "source_expert_id": 72, + "source_layer": 7, + "target_layer": 3 + }, + { + "expert_slot": 0, + "source_expert_id": 129, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 1, + "source_expert_id": 142, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 2, + "source_expert_id": 181, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 3, + "source_expert_id": 82, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 4, + "source_expert_id": 74, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 5, + "source_expert_id": 95, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 6, + "source_expert_id": 177, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 7, + "source_expert_id": 88, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 8, + "source_expert_id": 202, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 9, + "source_expert_id": 150, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 10, + "source_expert_id": 6, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 11, + "source_expert_id": 239, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 12, + "source_expert_id": 98, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 13, + "source_expert_id": 271, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 14, + "source_expert_id": 221, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 15, + "source_expert_id": 219, + "source_layer": 9, + "target_layer": 4 + }, + { + "expert_slot": 0, + "source_expert_id": 96, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 1, + "source_expert_id": 98, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 2, + "source_expert_id": 285, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 3, + "source_expert_id": 75, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 4, + "source_expert_id": 81, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 5, + "source_expert_id": 284, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 6, + "source_expert_id": 111, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 7, + "source_expert_id": 9, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 8, + "source_expert_id": 177, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 9, + "source_expert_id": 268, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 10, + "source_expert_id": 0, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 11, + "source_expert_id": 159, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 12, + "source_expert_id": 70, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 13, + "source_expert_id": 28, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 14, + "source_expert_id": 65, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 15, + "source_expert_id": 144, + "source_layer": 10, + "target_layer": 5 + }, + { + "expert_slot": 0, + "source_expert_id": 246, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 1, + "source_expert_id": 103, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 2, + "source_expert_id": 109, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 3, + "source_expert_id": 138, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 4, + "source_expert_id": 128, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 5, + "source_expert_id": 69, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 6, + "source_expert_id": 178, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 7, + "source_expert_id": 219, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 8, + "source_expert_id": 28, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 9, + "source_expert_id": 159, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 10, + "source_expert_id": 220, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 11, + "source_expert_id": 66, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 12, + "source_expert_id": 81, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 13, + "source_expert_id": 91, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 14, + "source_expert_id": 166, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 15, + "source_expert_id": 47, + "source_layer": 11, + "target_layer": 6 + }, + { + "expert_slot": 0, + "source_expert_id": 178, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 1, + "source_expert_id": 145, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 2, + "source_expert_id": 13, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 3, + "source_expert_id": 25, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 4, + "source_expert_id": 75, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 5, + "source_expert_id": 187, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 6, + "source_expert_id": 70, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 7, + "source_expert_id": 169, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 8, + "source_expert_id": 235, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 9, + "source_expert_id": 20, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 10, + "source_expert_id": 142, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 11, + "source_expert_id": 209, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 12, + "source_expert_id": 156, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 13, + "source_expert_id": 52, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 14, + "source_expert_id": 99, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 15, + "source_expert_id": 177, + "source_layer": 13, + "target_layer": 7 + }, + { + "expert_slot": 0, + "source_expert_id": 143, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 1, + "source_expert_id": 121, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 2, + "source_expert_id": 277, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 3, + "source_expert_id": 228, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 4, + "source_expert_id": 46, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 5, + "source_expert_id": 72, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 6, + "source_expert_id": 285, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 7, + "source_expert_id": 243, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 8, + "source_expert_id": 70, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 9, + "source_expert_id": 180, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 10, + "source_expert_id": 86, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 11, + "source_expert_id": 23, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 12, + "source_expert_id": 150, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 13, + "source_expert_id": 205, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 14, + "source_expert_id": 28, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 15, + "source_expert_id": 275, + "source_layer": 14, + "target_layer": 8 + }, + { + "expert_slot": 0, + "source_expert_id": 229, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 1, + "source_expert_id": 219, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 2, + "source_expert_id": 23, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 3, + "source_expert_id": 224, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 4, + "source_expert_id": 64, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 5, + "source_expert_id": 244, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 6, + "source_expert_id": 273, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 7, + "source_expert_id": 90, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 8, + "source_expert_id": 66, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 9, + "source_expert_id": 185, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 10, + "source_expert_id": 77, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 11, + "source_expert_id": 237, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 12, + "source_expert_id": 96, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 13, + "source_expert_id": 136, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 14, + "source_expert_id": 226, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 15, + "source_expert_id": 109, + "source_layer": 16, + "target_layer": 9 + }, + { + "expert_slot": 0, + "source_expert_id": 279, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 1, + "source_expert_id": 172, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 2, + "source_expert_id": 114, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 3, + "source_expert_id": 217, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 4, + "source_expert_id": 9, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 5, + "source_expert_id": 66, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 6, + "source_expert_id": 157, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 7, + "source_expert_id": 180, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 8, + "source_expert_id": 210, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 9, + "source_expert_id": 102, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 10, + "source_expert_id": 150, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 11, + "source_expert_id": 283, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 12, + "source_expert_id": 175, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 13, + "source_expert_id": 89, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 14, + "source_expert_id": 32, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 15, + "source_expert_id": 186, + "source_layer": 17, + "target_layer": 10 + }, + { + "expert_slot": 0, + "source_expert_id": 157, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 1, + "source_expert_id": 281, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 2, + "source_expert_id": 111, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 3, + "source_expert_id": 228, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 4, + "source_expert_id": 27, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 5, + "source_expert_id": 5, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 6, + "source_expert_id": 23, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 7, + "source_expert_id": 21, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 8, + "source_expert_id": 282, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 9, + "source_expert_id": 261, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 10, + "source_expert_id": 92, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 11, + "source_expert_id": 81, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 12, + "source_expert_id": 271, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 13, + "source_expert_id": 85, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 14, + "source_expert_id": 80, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 15, + "source_expert_id": 136, + "source_layer": 19, + "target_layer": 11 + }, + { + "expert_slot": 0, + "source_expert_id": 269, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 1, + "source_expert_id": 92, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 2, + "source_expert_id": 42, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 3, + "source_expert_id": 15, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 4, + "source_expert_id": 173, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 5, + "source_expert_id": 216, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 6, + "source_expert_id": 141, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 7, + "source_expert_id": 282, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 8, + "source_expert_id": 29, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 9, + "source_expert_id": 210, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 10, + "source_expert_id": 35, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 11, + "source_expert_id": 110, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 12, + "source_expert_id": 260, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 13, + "source_expert_id": 64, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 14, + "source_expert_id": 245, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 15, + "source_expert_id": 247, + "source_layer": 20, + "target_layer": 12 + }, + { + "expert_slot": 0, + "source_expert_id": 123, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 1, + "source_expert_id": 165, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 2, + "source_expert_id": 166, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 3, + "source_expert_id": 178, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 4, + "source_expert_id": 5, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 5, + "source_expert_id": 182, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 6, + "source_expert_id": 33, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 7, + "source_expert_id": 37, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 8, + "source_expert_id": 110, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 9, + "source_expert_id": 211, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 10, + "source_expert_id": 186, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 11, + "source_expert_id": 158, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 12, + "source_expert_id": 63, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 13, + "source_expert_id": 106, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 14, + "source_expert_id": 159, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 15, + "source_expert_id": 267, + "source_layer": 21, + "target_layer": 13 + }, + { + "expert_slot": 0, + "source_expert_id": 269, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 1, + "source_expert_id": 234, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 2, + "source_expert_id": 202, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 3, + "source_expert_id": 229, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 4, + "source_expert_id": 106, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 5, + "source_expert_id": 90, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 6, + "source_expert_id": 67, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 7, + "source_expert_id": 11, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 8, + "source_expert_id": 133, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 9, + "source_expert_id": 152, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 10, + "source_expert_id": 254, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 11, + "source_expert_id": 70, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 12, + "source_expert_id": 219, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 13, + "source_expert_id": 13, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 14, + "source_expert_id": 57, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 15, + "source_expert_id": 10, + "source_layer": 23, + "target_layer": 14 + }, + { + "expert_slot": 0, + "source_expert_id": 135, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 1, + "source_expert_id": 49, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 2, + "source_expert_id": 193, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 3, + "source_expert_id": 207, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 4, + "source_expert_id": 272, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 5, + "source_expert_id": 25, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 6, + "source_expert_id": 45, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 7, + "source_expert_id": 101, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 8, + "source_expert_id": 177, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 9, + "source_expert_id": 56, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 10, + "source_expert_id": 29, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 11, + "source_expert_id": 23, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 12, + "source_expert_id": 119, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 13, + "source_expert_id": 198, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 14, + "source_expert_id": 35, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 15, + "source_expert_id": 159, + "source_layer": 24, + "target_layer": 15 + }, + { + "expert_slot": 0, + "source_expert_id": 227, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 1, + "source_expert_id": 49, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 2, + "source_expert_id": 43, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 3, + "source_expert_id": 214, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 4, + "source_expert_id": 245, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 5, + "source_expert_id": 127, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 6, + "source_expert_id": 248, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 7, + "source_expert_id": 150, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 8, + "source_expert_id": 25, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 9, + "source_expert_id": 223, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 10, + "source_expert_id": 246, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 11, + "source_expert_id": 203, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 12, + "source_expert_id": 145, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 13, + "source_expert_id": 57, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 14, + "source_expert_id": 89, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 15, + "source_expert_id": 208, + "source_layer": 26, + "target_layer": 16 + }, + { + "expert_slot": 0, + "source_expert_id": 134, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 1, + "source_expert_id": 196, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 2, + "source_expert_id": 89, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 3, + "source_expert_id": 66, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 4, + "source_expert_id": 106, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 5, + "source_expert_id": 205, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 6, + "source_expert_id": 60, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 7, + "source_expert_id": 121, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 8, + "source_expert_id": 209, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 9, + "source_expert_id": 224, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 10, + "source_expert_id": 286, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 11, + "source_expert_id": 114, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 12, + "source_expert_id": 26, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 13, + "source_expert_id": 248, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 14, + "source_expert_id": 188, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 15, + "source_expert_id": 194, + "source_layer": 27, + "target_layer": 17 + }, + { + "expert_slot": 0, + "source_expert_id": 69, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 1, + "source_expert_id": 214, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 2, + "source_expert_id": 198, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 3, + "source_expert_id": 283, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 4, + "source_expert_id": 272, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 5, + "source_expert_id": 56, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 6, + "source_expert_id": 89, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 7, + "source_expert_id": 107, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 8, + "source_expert_id": 60, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 9, + "source_expert_id": 50, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 10, + "source_expert_id": 245, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 11, + "source_expert_id": 61, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 12, + "source_expert_id": 185, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 13, + "source_expert_id": 67, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 14, + "source_expert_id": 97, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 15, + "source_expert_id": 99, + "source_layer": 28, + "target_layer": 18 + }, + { + "expert_slot": 0, + "source_expert_id": 93, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 1, + "source_expert_id": 139, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 2, + "source_expert_id": 36, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 3, + "source_expert_id": 279, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 4, + "source_expert_id": 29, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 5, + "source_expert_id": 97, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 6, + "source_expert_id": 111, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 7, + "source_expert_id": 64, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 8, + "source_expert_id": 0, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 9, + "source_expert_id": 195, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 10, + "source_expert_id": 242, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 11, + "source_expert_id": 24, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 12, + "source_expert_id": 140, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 13, + "source_expert_id": 101, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 14, + "source_expert_id": 8, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 15, + "source_expert_id": 136, + "source_layer": 30, + "target_layer": 19 + }, + { + "expert_slot": 0, + "source_expert_id": 147, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 1, + "source_expert_id": 241, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 2, + "source_expert_id": 11, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 3, + "source_expert_id": 251, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 4, + "source_expert_id": 178, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 5, + "source_expert_id": 268, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 6, + "source_expert_id": 118, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 7, + "source_expert_id": 154, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 8, + "source_expert_id": 103, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 9, + "source_expert_id": 248, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 10, + "source_expert_id": 25, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 11, + "source_expert_id": 157, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 12, + "source_expert_id": 197, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 13, + "source_expert_id": 66, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 14, + "source_expert_id": 210, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 15, + "source_expert_id": 98, + "source_layer": 31, + "target_layer": 20 + }, + { + "expert_slot": 0, + "source_expert_id": 206, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 1, + "source_expert_id": 72, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 2, + "source_expert_id": 170, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 3, + "source_expert_id": 171, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 4, + "source_expert_id": 237, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 5, + "source_expert_id": 8, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 6, + "source_expert_id": 168, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 7, + "source_expert_id": 61, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 8, + "source_expert_id": 172, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 9, + "source_expert_id": 202, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 10, + "source_expert_id": 130, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 11, + "source_expert_id": 222, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 12, + "source_expert_id": 256, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 13, + "source_expert_id": 128, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 14, + "source_expert_id": 226, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 15, + "source_expert_id": 252, + "source_layer": 33, + "target_layer": 21 + }, + { + "expert_slot": 0, + "source_expert_id": 52, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 1, + "source_expert_id": 258, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 2, + "source_expert_id": 18, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 3, + "source_expert_id": 59, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 4, + "source_expert_id": 80, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 5, + "source_expert_id": 172, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 6, + "source_expert_id": 137, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 7, + "source_expert_id": 40, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 8, + "source_expert_id": 197, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 9, + "source_expert_id": 89, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 10, + "source_expert_id": 35, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 11, + "source_expert_id": 154, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 12, + "source_expert_id": 84, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 13, + "source_expert_id": 19, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 14, + "source_expert_id": 255, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 15, + "source_expert_id": 38, + "source_layer": 34, + "target_layer": 22 + }, + { + "expert_slot": 0, + "source_expert_id": 220, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 1, + "source_expert_id": 31, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 2, + "source_expert_id": 280, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 3, + "source_expert_id": 106, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 4, + "source_expert_id": 10, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 5, + "source_expert_id": 178, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 6, + "source_expert_id": 234, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 7, + "source_expert_id": 36, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 8, + "source_expert_id": 7, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 9, + "source_expert_id": 212, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 10, + "source_expert_id": 187, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 11, + "source_expert_id": 199, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 12, + "source_expert_id": 221, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 13, + "source_expert_id": 47, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 14, + "source_expert_id": 186, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 15, + "source_expert_id": 147, + "source_layer": 36, + "target_layer": 23 + }, + { + "expert_slot": 0, + "source_expert_id": 36, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 1, + "source_expert_id": 197, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 2, + "source_expert_id": 33, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 3, + "source_expert_id": 93, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 4, + "source_expert_id": 181, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 5, + "source_expert_id": 134, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 6, + "source_expert_id": 84, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 7, + "source_expert_id": 244, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 8, + "source_expert_id": 264, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 9, + "source_expert_id": 164, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 10, + "source_expert_id": 207, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 11, + "source_expert_id": 202, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 12, + "source_expert_id": 71, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 13, + "source_expert_id": 59, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 14, + "source_expert_id": 214, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 15, + "source_expert_id": 284, + "source_layer": 37, + "target_layer": 24 + }, + { + "expert_slot": 0, + "source_expert_id": 54, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 1, + "source_expert_id": 143, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 2, + "source_expert_id": 130, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 3, + "source_expert_id": 15, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 4, + "source_expert_id": 14, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 5, + "source_expert_id": 107, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 6, + "source_expert_id": 76, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 7, + "source_expert_id": 225, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 8, + "source_expert_id": 8, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 9, + "source_expert_id": 195, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 10, + "source_expert_id": 281, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 11, + "source_expert_id": 212, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 12, + "source_expert_id": 161, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 13, + "source_expert_id": 177, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 14, + "source_expert_id": 258, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 15, + "source_expert_id": 259, + "source_layer": 38, + "target_layer": 25 + }, + { + "expert_slot": 0, + "source_expert_id": 5, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 1, + "source_expert_id": 187, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 2, + "source_expert_id": 78, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 3, + "source_expert_id": 105, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 4, + "source_expert_id": 165, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 5, + "source_expert_id": 9, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 6, + "source_expert_id": 156, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 7, + "source_expert_id": 139, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 8, + "source_expert_id": 235, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 9, + "source_expert_id": 121, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 10, + "source_expert_id": 62, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 11, + "source_expert_id": 256, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 12, + "source_expert_id": 14, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 13, + "source_expert_id": 201, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 14, + "source_expert_id": 180, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 15, + "source_expert_id": 277, + "source_layer": 40, + "target_layer": 26 + }, + { + "expert_slot": 0, + "source_expert_id": 144, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 1, + "source_expert_id": 154, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 2, + "source_expert_id": 125, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 3, + "source_expert_id": 98, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 4, + "source_expert_id": 92, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 5, + "source_expert_id": 123, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 6, + "source_expert_id": 196, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 7, + "source_expert_id": 80, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 8, + "source_expert_id": 141, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 9, + "source_expert_id": 83, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 10, + "source_expert_id": 249, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 11, + "source_expert_id": 250, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 12, + "source_expert_id": 20, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 13, + "source_expert_id": 34, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 14, + "source_expert_id": 134, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 15, + "source_expert_id": 184, + "source_layer": 41, + "target_layer": 27 + }, + { + "expert_slot": 0, + "source_expert_id": 249, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 1, + "source_expert_id": 148, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 2, + "source_expert_id": 151, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 3, + "source_expert_id": 221, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 4, + "source_expert_id": 18, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 5, + "source_expert_id": 87, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 6, + "source_expert_id": 239, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 7, + "source_expert_id": 105, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 8, + "source_expert_id": 207, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 9, + "source_expert_id": 35, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 10, + "source_expert_id": 147, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 11, + "source_expert_id": 20, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 12, + "source_expert_id": 84, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 13, + "source_expert_id": 130, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 14, + "source_expert_id": 52, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 15, + "source_expert_id": 17, + "source_layer": 43, + "target_layer": 28 + }, + { + "expert_slot": 0, + "source_expert_id": 44, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 1, + "source_expert_id": 212, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 2, + "source_expert_id": 152, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 3, + "source_expert_id": 127, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 4, + "source_expert_id": 244, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 5, + "source_expert_id": 138, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 6, + "source_expert_id": 156, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 7, + "source_expert_id": 266, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 8, + "source_expert_id": 24, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 9, + "source_expert_id": 135, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 10, + "source_expert_id": 113, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 11, + "source_expert_id": 185, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 12, + "source_expert_id": 147, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 13, + "source_expert_id": 186, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 14, + "source_expert_id": 17, + "source_layer": 44, + "target_layer": 29 + }, + { + "expert_slot": 15, + "source_expert_id": 184, + "source_layer": 44, + "target_layer": 29 + } + ], + "float32_matmul_precision": "highest", + "fp8_format": "native torch E4M3 with weight_scale_inv", + "output_dtype": "bfloat16", + "platform": "Windows-11-10.0.26200-SP0", + "python_version": "3.12.10", + "script_path": "\\scripts\\glm_expert_pipeline.py", + "script_sha256": "f144611bdb294924558d9fa995f3d16360106f652f2835a9c6800fb58f30e785", + "tf32_allowed": false, + "torch_version": "2.12.0+cu130", + "verification": [ + { + "expert_slot": 0, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.944523572921753, + "relative_to_reference_max": 6.743565381870329e-07, + "seed": 1234, + "source_expert_id": 149, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023889047145843507, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.169741153717041e-06, + "passed": true, + "reference_max_absolute": 1.7859030961990356, + "relative_to_reference_max": 6.549857918980144e-07, + "seed": 1235, + "source_expert_id": 171, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023571806192398074, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.855207085609436, + "relative_to_reference_max": 6.425659457398039e-07, + "seed": 1236, + "source_expert_id": 186, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023710414171218873, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.731640338897705, + "relative_to_reference_max": 6.195764685407983e-07, + "seed": 1237, + "source_expert_id": 98, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002346328067779541, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.1101365089416504e-06, + "passed": true, + "reference_max_absolute": 1.5225051641464233, + "relative_to_reference_max": 7.29151227256452e-07, + "seed": 1238, + "source_expert_id": 286, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002304501032829285, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.7259011268615723, + "relative_to_reference_max": 6.043690842627264e-07, + "seed": 1239, + "source_expert_id": 189, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023451802253723147, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.4007091522216797e-06, + "passed": true, + "reference_max_absolute": 2.052611827850342, + "relative_to_reference_max": 6.824033327765697e-07, + "seed": 1240, + "source_expert_id": 12, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0024105223655700686, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.9211864471435547, + "relative_to_reference_max": 6.825481134370144e-07, + "seed": 1241, + "source_expert_id": 10, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002384237289428711, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.0356307029724121e-06, + "passed": true, + "reference_max_absolute": 1.9792548418045044, + "relative_to_reference_max": 5.23242728070438e-07, + "seed": 1242, + "source_expert_id": 202, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023958509683609007, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.8515863418579102, + "relative_to_reference_max": 7.082047190642014e-07, + "seed": 1243, + "source_expert_id": 177, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002370317268371582, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.491371512413025, + "relative_to_reference_max": 7.593602541730643e-07, + "seed": 1244, + "source_expert_id": 66, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0022982743024826053, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.9628398418426514, + "relative_to_reference_max": 6.073307001903496e-07, + "seed": 1245, + "source_expert_id": 24, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023925679683685303, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.5925530195236206, + "relative_to_reference_max": 7.485420459435646e-07, + "seed": 1246, + "source_expert_id": 118, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002318510603904724, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.7477917671203613, + "relative_to_reference_max": 6.650051768095303e-07, + "seed": 1247, + "source_expert_id": 107, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023495583534240724, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.7801982164382935, + "relative_to_reference_max": 6.026764862755497e-07, + "seed": 1248, + "source_expert_id": 220, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023560396432876587, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.3709068298339844e-06, + "passed": true, + "reference_max_absolute": 1.856876254081726, + "relative_to_reference_max": 7.382865857757084e-07, + "seed": 1249, + "source_expert_id": 47, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.0023713752508163453, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5192275047302246, + "relative_to_reference_max": 6.669698633198275e-07, + "seed": 1250, + "source_expert_id": 241, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0023038455009460447, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.4254485368728638, + "relative_to_reference_max": 7.73571195185895e-07, + "seed": 1251, + "source_expert_id": 243, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0022850897073745727, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.0281801223754883e-06, + "passed": true, + "reference_max_absolute": 1.3773361444473267, + "relative_to_reference_max": 7.46499049284776e-07, + "seed": 1252, + "source_expert_id": 173, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0022754672288894653, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.525824785232544, + "relative_to_reference_max": 8.203415964910006e-07, + "seed": 1253, + "source_expert_id": 268, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002305164957046509, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.6929163932800293, + "relative_to_reference_max": 6.865611188678826e-07, + "seed": 1254, + "source_expert_id": 39, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002338583278656006, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.296401023864746e-06, + "passed": true, + "reference_max_absolute": 1.4479269981384277, + "relative_to_reference_max": 8.953497141302733e-07, + "seed": 1255, + "source_expert_id": 45, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0022895853996276858, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.6275742053985596, + "relative_to_reference_max": 6.225700541460163e-07, + "seed": 1256, + "source_expert_id": 216, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002325514841079712, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.46029531955719, + "relative_to_reference_max": 7.347031737952603e-07, + "seed": 1257, + "source_expert_id": 197, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002292059063911438, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.7773276567459106, + "relative_to_reference_max": 6.371859721160711e-07, + "seed": 1258, + "source_expert_id": 205, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0023554655313491823, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 2.1613759994506836, + "relative_to_reference_max": 4.96389154977989e-07, + "seed": 1259, + "source_expert_id": 223, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002432275199890137, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.3245570659637451, + "relative_to_reference_max": 8.999936100453686e-07, + "seed": 1260, + "source_expert_id": 89, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002264911413192749, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.648701786994934, + "relative_to_reference_max": 7.230494349621733e-07, + "seed": 1261, + "source_expert_id": 178, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.002329740357398987, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.2218952178955078e-06, + "passed": true, + "reference_max_absolute": 1.8238903284072876, + "relative_to_reference_max": 6.699389754221286e-07, + "seed": 1262, + "source_expert_id": 248, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0023647780656814573, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.441463828086853, + "relative_to_reference_max": 6.616012818524836e-07, + "seed": 1263, + "source_expert_id": 93, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0022882927656173706, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 2.0251355171203613, + "relative_to_reference_max": 5.00351187668898e-07, + "seed": 1264, + "source_expert_id": 142, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0024050271034240724, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3000446557998657, + "relative_to_reference_max": 6.8772227757113e-07, + "seed": 1265, + "source_expert_id": 74, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0022600089311599733, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.7118138074874878, + "relative_to_reference_max": 7.660308494550909e-07, + "seed": 1266, + "source_expert_id": 160, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023423627614974976, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.5538097620010376, + "relative_to_reference_max": 6.137651723709618e-07, + "seed": 1267, + "source_expert_id": 276, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023107619524002075, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.5929484367370605, + "relative_to_reference_max": 6.548117060875786e-07, + "seed": 1268, + "source_expert_id": 104, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023185896873474123, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.407081127166748, + "relative_to_reference_max": 7.624888041227261e-07, + "seed": 1269, + "source_expert_id": 234, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0022814162254333496, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.6671451330184937, + "relative_to_reference_max": 5.899166301216294e-07, + "seed": 1270, + "source_expert_id": 16, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.002333429026603699, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.7341564893722534, + "relative_to_reference_max": 5.241573285890393e-07, + "seed": 1271, + "source_expert_id": 83, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023468312978744505, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.566731572151184, + "relative_to_reference_max": 6.847909527245435e-07, + "seed": 1272, + "source_expert_id": 209, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.002313346314430237, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.2814998626708984e-06, + "passed": true, + "reference_max_absolute": 1.5879408121109009, + "relative_to_reference_max": 8.070199171764843e-07, + "seed": 1273, + "source_expert_id": 260, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.00231758816242218, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.469634771347046, + "relative_to_reference_max": 7.908703548533888e-07, + "seed": 1274, + "source_expert_id": 270, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.002293926954269409, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.9720240831375122, + "relative_to_reference_max": 5.440519794515195e-07, + "seed": 1275, + "source_expert_id": 282, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023944048166275023, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.164214134216309e-07, + "passed": true, + "reference_max_absolute": 1.3675719499588013, + "relative_to_reference_max": 6.701083723231077e-07, + "seed": 1276, + "source_expert_id": 55, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.00227351438999176, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5638099908828735, + "relative_to_reference_max": 6.479552932192089e-07, + "seed": 1277, + "source_expert_id": 174, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.002312761998176575, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.087784767150879e-06, + "passed": true, + "reference_max_absolute": 1.6437931060791016, + "relative_to_reference_max": 6.617528465887928e-07, + "seed": 1278, + "source_expert_id": 236, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023287586212158203, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.4655778408050537, + "relative_to_reference_max": 7.523898749308411e-07, + "seed": 1279, + "source_expert_id": 256, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.002293115568161011, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.760260581970215e-07, + "passed": true, + "reference_max_absolute": 1.4268778562545776, + "relative_to_reference_max": 6.8402915772973e-07, + "seed": 1280, + "source_expert_id": 137, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0022853755712509156, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.5255913734436035, + "relative_to_reference_max": 7.618623134296869e-07, + "seed": 1281, + "source_expert_id": 206, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0023051182746887207, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.7494051456451416, + "relative_to_reference_max": 5.621777443847983e-07, + "seed": 1282, + "source_expert_id": 81, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023498810291290285, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.605658769607544, + "relative_to_reference_max": 5.939458211531131e-07, + "seed": 1283, + "source_expert_id": 57, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023211317539215087, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1175870895385742e-06, + "passed": true, + "reference_max_absolute": 1.5149204730987549, + "relative_to_reference_max": 7.37719972357731e-07, + "seed": 1284, + "source_expert_id": 131, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002302984094619751, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4547488689422607, + "relative_to_reference_max": 6.555594142511078e-07, + "seed": 1285, + "source_expert_id": 233, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002290949773788452, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.2665987014770508e-06, + "passed": true, + "reference_max_absolute": 1.921863079071045, + "relative_to_reference_max": 6.590473146969846e-07, + "seed": 1286, + "source_expert_id": 68, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002384372615814209, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.8868228197097778, + "relative_to_reference_max": 5.370292062386115e-07, + "seed": 1287, + "source_expert_id": 129, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023773645639419555, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.7153898477554321, + "relative_to_reference_max": 4.864579488722967e-07, + "seed": 1288, + "source_expert_id": 94, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023430779695510864, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0970979928970337e-06, + "passed": true, + "reference_max_absolute": 1.5693703889846802, + "relative_to_reference_max": 6.990688753894562e-07, + "seed": 1289, + "source_expert_id": 145, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002313874077796936, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4844233989715576, + "relative_to_reference_max": 6.223776819057382e-07, + "seed": 1290, + "source_expert_id": 208, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0022968846797943115, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 8.866190910339355e-07, + "passed": true, + "reference_max_absolute": 1.470833420753479, + "relative_to_reference_max": 6.028004793226265e-07, + "seed": 1291, + "source_expert_id": 262, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002294166684150696, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3203600645065308, + "relative_to_reference_max": 7.448548810520926e-07, + "seed": 1292, + "source_expert_id": 184, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002264072012901306, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.6981983184814453, + "relative_to_reference_max": 6.142281924422556e-07, + "seed": 1293, + "source_expert_id": 48, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002339639663696289, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.3606081008911133, + "relative_to_reference_max": 7.666287470184712e-07, + "seed": 1294, + "source_expert_id": 211, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002272121620178223, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.614612340927124, + "relative_to_reference_max": 5.906521907658913e-07, + "seed": 1295, + "source_expert_id": 51, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023229224681854247, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.5119847059249878, + "relative_to_reference_max": 6.011772667162168e-07, + "seed": 1296, + "source_expert_id": 44, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0023023969411849975, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.8392598628997803, + "relative_to_reference_max": 6.319338537011556e-07, + "seed": 1297, + "source_expert_id": 72, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.002367851972579956, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.8250094652175903, + "relative_to_reference_max": 6.531981988190308e-07, + "seed": 1298, + "source_expert_id": 129, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023650018930435183, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.426945686340332, + "relative_to_reference_max": 8.771865336328355e-07, + "seed": 1299, + "source_expert_id": 142, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0022853891372680664, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.6650640964508057, + "relative_to_reference_max": 6.622483366827587e-07, + "seed": 1300, + "source_expert_id": 181, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002333012819290161, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.486488938331604, + "relative_to_reference_max": 7.217568717068339e-07, + "seed": 1301, + "source_expert_id": 82, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002297297787666321, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.1995434761047363e-06, + "passed": true, + "reference_max_absolute": 1.3453524112701416, + "relative_to_reference_max": 8.916202669694941e-07, + "seed": 1302, + "source_expert_id": 74, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002269070482254028, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.087784767150879e-06, + "passed": true, + "reference_max_absolute": 1.3518139123916626, + "relative_to_reference_max": 8.046852878044015e-07, + "seed": 1303, + "source_expert_id": 95, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0022703627824783327, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.98377799987793e-07, + "passed": true, + "reference_max_absolute": 1.5364389419555664, + "relative_to_reference_max": 6.497998538861988e-07, + "seed": 1304, + "source_expert_id": 177, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023072877883911136, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.1830971240997314, + "relative_to_reference_max": 7.808927730439442e-07, + "seed": 1305, + "source_expert_id": 88, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002236619424819946, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.479083776473999, + "relative_to_reference_max": 6.850720542667335e-07, + "seed": 1306, + "source_expert_id": 202, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0022958167552948, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.4012539386749268, + "relative_to_reference_max": 7.443913303506636e-07, + "seed": 1307, + "source_expert_id": 150, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0022802507877349855, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.5930695533752441, + "relative_to_reference_max": 5.799319885695779e-07, + "seed": 1308, + "source_expert_id": 6, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002318613910675049, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.6284228563308716, + "relative_to_reference_max": 6.222456023890132e-07, + "seed": 1309, + "source_expert_id": 239, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023256845712661743, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.0505318641662598e-06, + "passed": true, + "reference_max_absolute": 1.5907105207443237, + "relative_to_reference_max": 6.604167448862385e-07, + "seed": 1310, + "source_expert_id": 98, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.002318142104148865, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.5085432529449463, + "relative_to_reference_max": 6.124265858569128e-07, + "seed": 1311, + "source_expert_id": 271, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023017086505889893, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.1473894119262695e-06, + "passed": true, + "reference_max_absolute": 1.583365797996521, + "relative_to_reference_max": 7.246521387402045e-07, + "seed": 1312, + "source_expert_id": 221, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023166731595993043, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.5270805358886719, + "relative_to_reference_max": 6.342661404144237e-07, + "seed": 1313, + "source_expert_id": 219, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.0023054161071777344, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 2.0291028022766113, + "relative_to_reference_max": 5.874975354478388e-07, + "seed": 1314, + "source_expert_id": 96, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0024058205604553223, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.527121901512146, + "relative_to_reference_max": 7.415833992106586e-07, + "seed": 1315, + "source_expert_id": 98, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0023054243803024292, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.7189884185791016, + "relative_to_reference_max": 6.414737391053487e-07, + "seed": 1316, + "source_expert_id": 285, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0023437976837158204, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.615107536315918, + "relative_to_reference_max": 6.089233172902337e-07, + "seed": 1317, + "source_expert_id": 75, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0023230215072631835, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.8118892908096313, + "relative_to_reference_max": 6.579281093797586e-07, + "seed": 1318, + "source_expert_id": 81, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002362377858161926, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4663959741592407, + "relative_to_reference_max": 6.909995519884081e-07, + "seed": 1319, + "source_expert_id": 284, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002293279194831848, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.7031301259994507, + "relative_to_reference_max": 6.649452284615317e-07, + "seed": 1320, + "source_expert_id": 111, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.00234062602519989, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.7885892391204834, + "relative_to_reference_max": 4.998742316433185e-07, + "seed": 1321, + "source_expert_id": 9, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0023577178478240966, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.2641501426696777, + "relative_to_reference_max": 7.779745503307099e-07, + "seed": 1322, + "source_expert_id": 177, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0022528300285339356, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4339381456375122, + "relative_to_reference_max": 6.650735384281569e-07, + "seed": 1323, + "source_expert_id": 268, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0022867876291275025, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.0281801223754883e-06, + "passed": true, + "reference_max_absolute": 1.5376983880996704, + "relative_to_reference_max": 6.686487612477381e-07, + "seed": 1324, + "source_expert_id": 0, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002307539677619934, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.5602128505706787, + "relative_to_reference_max": 6.876520761667889e-07, + "seed": 1325, + "source_expert_id": 159, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002312042570114136, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.5646719932556152, + "relative_to_reference_max": 7.237863626459191e-07, + "seed": 1326, + "source_expert_id": 70, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002312934398651123, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0281801223754883e-06, + "passed": true, + "reference_max_absolute": 1.6899970769882202, + "relative_to_reference_max": 6.083916572257213e-07, + "seed": 1327, + "source_expert_id": 28, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002337999415397644, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.5929450988769531, + "relative_to_reference_max": 7.483578036357019e-07, + "seed": 1328, + "source_expert_id": 65, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0023185890197753908, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.521106243133545, + "relative_to_reference_max": 7.445162070990169e-07, + "seed": 1329, + "source_expert_id": 144, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.002304221248626709, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5120958089828491, + "relative_to_reference_max": 6.70115580746996e-07, + "seed": 1330, + "source_expert_id": 246, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.00230241916179657, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.6092475652694702, + "relative_to_reference_max": 6.296600865212221e-07, + "seed": 1331, + "source_expert_id": 103, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.002321849513053894, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.2956594228744507, + "relative_to_reference_max": 6.900499126902847e-07, + "seed": 1332, + "source_expert_id": 109, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.00225913188457489, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.6178292036056519, + "relative_to_reference_max": 7.368471856305956e-07, + "seed": 1333, + "source_expert_id": 138, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023235658407211304, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.462777853012085, + "relative_to_reference_max": 6.927087110972751e-07, + "seed": 1334, + "source_expert_id": 128, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.002292555570602417, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.613714575767517, + "relative_to_reference_max": 6.648533898547362e-07, + "seed": 1335, + "source_expert_id": 69, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023227429151535034, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.4991499185562134, + "relative_to_reference_max": 7.057215770535444e-07, + "seed": 1336, + "source_expert_id": 178, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.002299829983711243, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.7813795804977417, + "relative_to_reference_max": 5.688169844736385e-07, + "seed": 1337, + "source_expert_id": 219, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023562759160995483, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.6367197036743164, + "relative_to_reference_max": 5.644656149397661e-07, + "seed": 1338, + "source_expert_id": 28, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023273439407348632, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.5021865367889404, + "relative_to_reference_max": 6.05098508450032e-07, + "seed": 1339, + "source_expert_id": 159, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023004373073577883, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3953051567077637, + "relative_to_reference_max": 6.407699902295392e-07, + "seed": 1340, + "source_expert_id": 220, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.002279061031341553, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.711003065109253, + "relative_to_reference_max": 6.270494938526158e-07, + "seed": 1341, + "source_expert_id": 66, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023422006130218507, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.2230615615844727, + "relative_to_reference_max": 7.066425569973013e-07, + "seed": 1342, + "source_expert_id": 81, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0022446123123168946, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.6136126518249512, + "relative_to_reference_max": 6.648953853600674e-07, + "seed": 1343, + "source_expert_id": 91, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023227225303649904, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.332909345626831, + "relative_to_reference_max": 7.937392353308493e-07, + "seed": 1344, + "source_expert_id": 166, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0022665818691253664, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.723641276359558, + "relative_to_reference_max": 5.705808118445107e-07, + "seed": 1345, + "source_expert_id": 47, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.0023447282552719116, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.5799146890640259, + "relative_to_reference_max": 5.281709402612258e-07, + "seed": 1346, + "source_expert_id": 178, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002315982937812805, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1194497346878052e-06, + "passed": true, + "reference_max_absolute": 1.3646280765533447, + "relative_to_reference_max": 8.203332130723933e-07, + "seed": 1347, + "source_expert_id": 145, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002272925615310669, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4539135694503784, + "relative_to_reference_max": 6.354380435198807e-07, + "seed": 1348, + "source_expert_id": 13, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002290782713890076, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5573716163635254, + "relative_to_reference_max": 5.740888444586649e-07, + "seed": 1349, + "source_expert_id": 25, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002311474323272705, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.2740590572357178, + "relative_to_reference_max": 6.783573683925887e-07, + "seed": 1350, + "source_expert_id": 75, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022548118114471436, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3000786304473877, + "relative_to_reference_max": 6.877043054874219e-07, + "seed": 1351, + "source_expert_id": 187, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022600157260894774, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.405796766281128, + "relative_to_reference_max": 6.889868442096019e-07, + "seed": 1352, + "source_expert_id": 70, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022811593532562255, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.7448197603225708, + "relative_to_reference_max": 6.148965241880821e-07, + "seed": 1353, + "source_expert_id": 169, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0023489639520645144, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.449183464050293, + "relative_to_reference_max": 6.992068197836375e-07, + "seed": 1354, + "source_expert_id": 235, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022898366928100584, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3551658391952515, + "relative_to_reference_max": 6.597492688878519e-07, + "seed": 1355, + "source_expert_id": 20, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022710331678390504, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.6716896295547485, + "relative_to_reference_max": 5.704852740279662e-07, + "seed": 1356, + "source_expert_id": 142, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0023343379259109496, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.539659857749939, + "relative_to_reference_max": 6.581187111434132e-07, + "seed": 1357, + "source_expert_id": 209, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002307931971549988, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.661523699760437, + "relative_to_reference_max": 7.174696910310047e-07, + "seed": 1358, + "source_expert_id": 156, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0023323047399520873, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.3904178142547607, + "relative_to_reference_max": 7.501926923515497e-07, + "seed": 1359, + "source_expert_id": 52, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.0022780835628509522, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.909272193908691e-07, + "passed": true, + "reference_max_absolute": 1.420554518699646, + "relative_to_reference_max": 6.975636671079325e-07, + "seed": 1360, + "source_expert_id": 99, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002284110903739929, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.4564872980117798, + "relative_to_reference_max": 7.366240731529908e-07, + "seed": 1361, + "source_expert_id": 177, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002291297459602356, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.2083600759506226, + "relative_to_reference_max": 6.905764626483469e-07, + "seed": 1362, + "source_expert_id": 143, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022416720151901244, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5649878978729248, + "relative_to_reference_max": 5.712949428209935e-07, + "seed": 1363, + "source_expert_id": 121, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.002312997579574585, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.65743088722229, + "relative_to_reference_max": 6.113551816810945e-07, + "seed": 1364, + "source_expert_id": 277, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0023314861774444582, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0505318641662598e-06, + "passed": true, + "reference_max_absolute": 1.754145622253418, + "relative_to_reference_max": 5.988852070426861e-07, + "seed": 1365, + "source_expert_id": 228, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0023508291244506837, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.584388256072998, + "relative_to_reference_max": 6.01919581738121e-07, + "seed": 1366, + "source_expert_id": 46, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0023168776512145998, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3562042713165283, + "relative_to_reference_max": 7.251685159782312e-07, + "seed": 1367, + "source_expert_id": 72, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.002271240854263306, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.195638656616211e-07, + "passed": true, + "reference_max_absolute": 1.480799913406372, + "relative_to_reference_max": 5.53460233379086e-07, + "seed": 1368, + "source_expert_id": 285, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022961599826812743, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.2525209188461304, + "relative_to_reference_max": 8.446824550745524e-07, + "seed": 1369, + "source_expert_id": 243, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.002250504183769226, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5407049655914307, + "relative_to_reference_max": 5.802990784076903e-07, + "seed": 1370, + "source_expert_id": 70, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.002308140993118286, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 8.791685104370117e-07, + "passed": true, + "reference_max_absolute": 1.350254774093628, + "relative_to_reference_max": 6.51113054591614e-07, + "seed": 1371, + "source_expert_id": 180, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.002270050954818726, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4704463481903076, + "relative_to_reference_max": 6.080260410257059e-07, + "seed": 1372, + "source_expert_id": 86, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022940892696380615, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.0693308115005493, + "relative_to_reference_max": 7.803619028656784e-07, + "seed": 1373, + "source_expert_id": 23, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.00221386616230011, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.404148817062378, + "relative_to_reference_max": 6.791832210501972e-07, + "seed": 1374, + "source_expert_id": 150, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022808297634124754, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3810393810272217, + "relative_to_reference_max": 6.905482417864898e-07, + "seed": 1375, + "source_expert_id": 205, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022762078762054443, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.087784767150879e-06, + "passed": true, + "reference_max_absolute": 1.4693132638931274, + "relative_to_reference_max": 7.403354981419404e-07, + "seed": 1376, + "source_expert_id": 28, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022938626527786257, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.98377799987793e-07, + "passed": true, + "reference_max_absolute": 1.2899296283721924, + "relative_to_reference_max": 7.739785008641759e-07, + "seed": 1377, + "source_expert_id": 275, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0022579859256744387, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.3437105417251587, + "relative_to_reference_max": 5.766572174727846e-07, + "seed": 1378, + "source_expert_id": 229, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.002268742108345032, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.5371149778366089, + "relative_to_reference_max": 6.010428675406218e-07, + "seed": 1379, + "source_expert_id": 219, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0023074229955673217, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3681281805038452, + "relative_to_reference_max": 7.188483161217811e-07, + "seed": 1380, + "source_expert_id": 23, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.002273625636100769, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.415740728378296, + "relative_to_reference_max": 7.578249212240995e-07, + "seed": 1381, + "source_expert_id": 224, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.002283148145675659, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4722365140914917, + "relative_to_reference_max": 6.07286711797516e-07, + "seed": 1382, + "source_expert_id": 64, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022944473028182985, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.015202522277832e-07, + "passed": true, + "reference_max_absolute": 1.587270736694336, + "relative_to_reference_max": 5.679687978783615e-07, + "seed": 1383, + "source_expert_id": 244, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0023174541473388673, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4521700143814087, + "relative_to_reference_max": 6.156783729016142e-07, + "seed": 1384, + "source_expert_id": 273, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022904340028762817, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 7.450580596923828e-07, + "passed": true, + "reference_max_absolute": 1.2562891244888306, + "relative_to_reference_max": 5.930625722765357e-07, + "seed": 1385, + "source_expert_id": 90, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.002251257824897766, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.4100133180618286, + "relative_to_reference_max": 5.91813578046557e-07, + "seed": 1386, + "source_expert_id": 66, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022820026636123656, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.30827796459198, + "relative_to_reference_max": 8.656327488368402e-07, + "seed": 1387, + "source_expert_id": 185, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022616555929183962, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4456936120986938, + "relative_to_reference_max": 7.008946796898945e-07, + "seed": 1388, + "source_expert_id": 77, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022891387224197386, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.417978048324585, + "relative_to_reference_max": 7.14594250862263e-07, + "seed": 1389, + "source_expert_id": 237, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.002283595609664917, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.315558671951294, + "relative_to_reference_max": 7.249196381273658e-07, + "seed": 1390, + "source_expert_id": 96, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022631117343902586, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.2906413078308105, + "relative_to_reference_max": 8.08188361274791e-07, + "seed": 1391, + "source_expert_id": 136, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022581282615661623, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.4421645402908325, + "relative_to_reference_max": 6.509473288138714e-07, + "seed": 1392, + "source_expert_id": 226, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0022884329080581664, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.532763123512268, + "relative_to_reference_max": 6.416364170742484e-07, + "seed": 1393, + "source_expert_id": 109, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0023065526247024537, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.64919114112854, + "relative_to_reference_max": 5.963387834602077e-07, + "seed": 1394, + "source_expert_id": 279, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.002329838228225708, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.7018414735794067, + "relative_to_reference_max": 5.954014970915337e-07, + "seed": 1395, + "source_expert_id": 172, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0023403682947158813, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.493661880493164e-07, + "passed": true, + "reference_max_absolute": 1.3358757495880127, + "relative_to_reference_max": 6.358122664561154e-07, + "seed": 1396, + "source_expert_id": 114, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0022671751499176024, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.3015724420547485, + "relative_to_reference_max": 8.471952022923546e-07, + "seed": 1397, + "source_expert_id": 217, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.00226031448841095, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.493661880493164e-07, + "passed": true, + "reference_max_absolute": 1.2503560781478882, + "relative_to_reference_max": 6.792994434892938e-07, + "seed": 1398, + "source_expert_id": 9, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0022500712156295778, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.3406925201416016, + "relative_to_reference_max": 6.446424786138746e-07, + "seed": 1399, + "source_expert_id": 66, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0022681385040283202, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.0505318641662598e-06, + "passed": true, + "reference_max_absolute": 1.5487968921661377, + "relative_to_reference_max": 6.782889799688276e-07, + "seed": 1400, + "source_expert_id": 157, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0023097593784332277, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.610047459602356, + "relative_to_reference_max": 5.182859808750992e-07, + "seed": 1401, + "source_expert_id": 180, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0023220094919204712, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.7010797262191772, + "relative_to_reference_max": 4.90550215838591e-07, + "seed": 1402, + "source_expert_id": 210, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0023402159452438356, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.3596800565719604, + "relative_to_reference_max": 7.452333777229403e-07, + "seed": 1403, + "source_expert_id": 102, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.002271936011314392, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.791685104370117e-07, + "passed": true, + "reference_max_absolute": 1.200415849685669, + "relative_to_reference_max": 7.323866230750149e-07, + "seed": 1404, + "source_expert_id": 150, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0022400831699371337, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.5489994287490845, + "relative_to_reference_max": 6.926300849726284e-07, + "seed": 1405, + "source_expert_id": 283, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0023097998857498168, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 6.556510925292969e-07, + "passed": true, + "reference_max_absolute": 1.1503360271453857, + "relative_to_reference_max": 5.699648424959154e-07, + "seed": 1406, + "source_expert_id": 175, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.002230067205429077, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 7.599592208862305e-07, + "passed": true, + "reference_max_absolute": 1.0732804536819458, + "relative_to_reference_max": 7.080714255804714e-07, + "seed": 1407, + "source_expert_id": 89, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.002214656090736389, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.1175870895385742e-06, + "passed": true, + "reference_max_absolute": 1.2569636106491089, + "relative_to_reference_max": 8.891165027135836e-07, + "seed": 1408, + "source_expert_id": 32, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0022513927221298217, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.121132850646973e-07, + "passed": true, + "reference_max_absolute": 1.434814691543579, + "relative_to_reference_max": 5.660056938718844e-07, + "seed": 1409, + "source_expert_id": 186, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.002286962938308716, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.8524426221847534, + "relative_to_reference_max": 5.148198950861406e-07, + "seed": 1410, + "source_expert_id": 157, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.002370488524436951, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.515215277671814, + "relative_to_reference_max": 5.900611515775049e-07, + "seed": 1411, + "source_expert_id": 281, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.002303043055534363, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3472191095352173, + "relative_to_reference_max": 6.19398152052159e-07, + "seed": 1412, + "source_expert_id": 111, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022694438219070436, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.298820972442627, + "relative_to_reference_max": 7.45734399236368e-07, + "seed": 1413, + "source_expert_id": 228, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022597641944885256, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3753142356872559, + "relative_to_reference_max": 6.06744993400348e-07, + "seed": 1414, + "source_expert_id": 27, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022750628471374513, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.493661880493164e-07, + "passed": true, + "reference_max_absolute": 1.3977774381637573, + "relative_to_reference_max": 6.076548131762078e-07, + "seed": 1415, + "source_expert_id": 5, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022795554876327514, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.217730164527893, + "relative_to_reference_max": 6.363153386945022e-07, + "seed": 1416, + "source_expert_id": 23, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022435460329055788, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.359810471534729, + "relative_to_reference_max": 6.794123249954392e-07, + "seed": 1417, + "source_expert_id": 21, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022719620943069457, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.2783297300338745, + "relative_to_reference_max": 7.227180689868434e-07, + "seed": 1418, + "source_expert_id": 282, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.002255665946006775, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.303375005722046, + "relative_to_reference_max": 6.859650274907344e-07, + "seed": 1419, + "source_expert_id": 261, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022606750011444094, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 7.599592208862305e-07, + "passed": true, + "reference_max_absolute": 1.2318799495697021, + "relative_to_reference_max": 6.169101308546223e-07, + "seed": 1420, + "source_expert_id": 92, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022463759899139404, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.3196417093276978, + "relative_to_reference_max": 5.871748191975813e-07, + "seed": 1421, + "source_expert_id": 81, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022639283418655395, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3064237833023071, + "relative_to_reference_max": 7.299884835191869e-07, + "seed": 1422, + "source_expert_id": 271, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0022612847566604614, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 7.897615432739258e-07, + "passed": true, + "reference_max_absolute": 1.5554057359695435, + "relative_to_reference_max": 5.077527522306824e-07, + "seed": 1423, + "source_expert_id": 85, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0023110811471939085, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.5321323871612549, + "relative_to_reference_max": 6.321747948913741e-07, + "seed": 1424, + "source_expert_id": 80, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.002306426477432251, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5450698137283325, + "relative_to_reference_max": 6.55814353615871e-07, + "seed": 1425, + "source_expert_id": 136, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.0023090139627456666, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.6159052848815918, + "relative_to_reference_max": 5.901795886979428e-07, + "seed": 1426, + "source_expert_id": 269, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0023231810569763183, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4664791822433472, + "relative_to_reference_max": 6.096708923362662e-07, + "seed": 1427, + "source_expert_id": 92, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022932958364486693, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.2367963790893555e-06, + "passed": true, + "reference_max_absolute": 1.9751849174499512, + "relative_to_reference_max": 6.261673872470194e-07, + "seed": 1428, + "source_expert_id": 42, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0023950369834899905, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.499490261077881e-07, + "passed": true, + "reference_max_absolute": 1.1236622333526611, + "relative_to_reference_max": 8.45404426625102e-07, + "seed": 1429, + "source_expert_id": 15, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022247324466705323, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3734867572784424, + "relative_to_reference_max": 6.509488838483264e-07, + "seed": 1430, + "source_expert_id": 173, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022746973514556883, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0207295417785645e-06, + "passed": true, + "reference_max_absolute": 1.7258347272872925, + "relative_to_reference_max": 5.914410723342966e-07, + "seed": 1431, + "source_expert_id": 216, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0023451669454574585, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.4306886196136475, + "relative_to_reference_max": 5.415995985830422e-07, + "seed": 1432, + "source_expert_id": 141, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022861377239227296, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.301084280014038, + "relative_to_reference_max": 7.329842740056473e-07, + "seed": 1433, + "source_expert_id": 282, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022602168560028075, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.4502506256103516, + "relative_to_reference_max": 5.753936679077634e-07, + "seed": 1434, + "source_expert_id": 29, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022900501251220703, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.5305213928222656, + "relative_to_reference_max": 5.938961958242049e-07, + "seed": 1435, + "source_expert_id": 210, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0023061042785644534, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.36678946018219, + "relative_to_reference_max": 6.541385470675798e-07, + "seed": 1436, + "source_expert_id": 35, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.002273357892036438, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.392807126045227, + "relative_to_reference_max": 5.991246104727154e-07, + "seed": 1437, + "source_expert_id": 110, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022785614252090456, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4360129833221436, + "relative_to_reference_max": 7.056196378095906e-07, + "seed": 1438, + "source_expert_id": 260, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022872025966644287, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0058283805847168e-06, + "passed": true, + "reference_max_absolute": 1.3509767055511475, + "relative_to_reference_max": 7.445194106247574e-07, + "seed": 1439, + "source_expert_id": 64, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022701953411102295, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4294241666793823, + "relative_to_reference_max": 7.088721352287851e-07, + "seed": 1440, + "source_expert_id": 245, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0022858848333358766, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.4881207942962646, + "relative_to_reference_max": 7.209654015112396e-07, + "seed": 1441, + "source_expert_id": 247, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.002297624158859253, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.1511147022247314e-06, + "passed": true, + "reference_max_absolute": 2.0546460151672363, + "relative_to_reference_max": 5.602496457916802e-07, + "seed": 1442, + "source_expert_id": 123, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0024109292030334473, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.791685104370117e-07, + "passed": true, + "reference_max_absolute": 1.5135315656661987, + "relative_to_reference_max": 5.808722661492926e-07, + "seed": 1443, + "source_expert_id": 165, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0023027063131332397, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.3140997886657715, + "relative_to_reference_max": 7.93761092244292e-07, + "seed": 1444, + "source_expert_id": 166, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.002262819957733154, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0319054126739502e-06, + "passed": true, + "reference_max_absolute": 1.4737012386322021, + "relative_to_reference_max": 7.002134392122114e-07, + "seed": 1445, + "source_expert_id": 178, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022947402477264405, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.2423350811004639, + "relative_to_reference_max": 6.716907858033739e-07, + "seed": 1446, + "source_expert_id": 5, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022484670162200926, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0281801223754883e-06, + "passed": true, + "reference_max_absolute": 2.019505739212036, + "relative_to_reference_max": 5.09124635009287e-07, + "seed": 1447, + "source_expert_id": 182, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.002403901147842407, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4784574508666992, + "relative_to_reference_max": 6.248891325732532e-07, + "seed": 1448, + "source_expert_id": 33, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022956914901733397, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.462237358093262e-07, + "passed": true, + "reference_max_absolute": 1.3450133800506592, + "relative_to_reference_max": 7.035050727701216e-07, + "seed": 1449, + "source_expert_id": 37, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.002269002676010132, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.2076655626296997, + "relative_to_reference_max": 6.416183470470207e-07, + "seed": 1450, + "source_expert_id": 110, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.00224153311252594, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4159270524978638, + "relative_to_reference_max": 6.735335091760942e-07, + "seed": 1451, + "source_expert_id": 211, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022831854104995728, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.080334186553955e-06, + "passed": true, + "reference_max_absolute": 2.0492498874664307, + "relative_to_reference_max": 5.271851876930516e-07, + "seed": 1452, + "source_expert_id": 186, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0024098499774932863, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.791685104370117e-07, + "passed": true, + "reference_max_absolute": 1.2186734676361084, + "relative_to_reference_max": 7.214143359848123e-07, + "seed": 1453, + "source_expert_id": 158, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022437346935272216, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.419156074523926e-07, + "passed": true, + "reference_max_absolute": 1.242857575416565, + "relative_to_reference_max": 6.77403126557128e-07, + "seed": 1454, + "source_expert_id": 63, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022485715150833132, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5157989263534546, + "relative_to_reference_max": 6.68478479279094e-07, + "seed": 1455, + "source_expert_id": 106, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.002303159785270691, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.3874510526657104, + "relative_to_reference_max": 6.658771797704279e-07, + "seed": 1456, + "source_expert_id": 159, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.002277490210533142, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.3088926076889038, + "relative_to_reference_max": 7.05842472171819e-07, + "seed": 1457, + "source_expert_id": 267, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0022617785215377807, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3624616861343384, + "relative_to_reference_max": 6.562163771133777e-07, + "seed": 1458, + "source_expert_id": 269, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022724923372268676, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.294599294662476e-07, + "passed": true, + "reference_max_absolute": 1.6250911951065063, + "relative_to_reference_max": 5.719432437176745e-07, + "seed": 1459, + "source_expert_id": 234, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0023250182390213013, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3659149408340454, + "relative_to_reference_max": 6.109201985490645e-07, + "seed": 1460, + "source_expert_id": 202, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002273182988166809, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.3395745754241943, + "relative_to_reference_max": 7.564184777549784e-07, + "seed": 1461, + "source_expert_id": 229, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002267914915084839, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.7133270502090454, + "relative_to_reference_max": 5.479240843701506e-07, + "seed": 1462, + "source_expert_id": 106, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002342665410041809, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.6234105825424194, + "relative_to_reference_max": 7.343138626341087e-07, + "seed": 1463, + "source_expert_id": 90, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002324682116508484, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.320580005645752, + "relative_to_reference_max": 6.99595624701882e-07, + "seed": 1464, + "source_expert_id": 67, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022641160011291506, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.588876724243164, + "relative_to_reference_max": 7.127603000615825e-07, + "seed": 1465, + "source_expert_id": 11, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002317775344848633, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.4504059553146362, + "relative_to_reference_max": 5.958796198238711e-07, + "seed": 1466, + "source_expert_id": 133, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022900811910629274, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 8.046627044677734e-07, + "passed": true, + "reference_max_absolute": 1.128697395324707, + "relative_to_reference_max": 7.129126972391795e-07, + "seed": 1467, + "source_expert_id": 152, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022257394790649414, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3378715515136719, + "relative_to_reference_max": 6.23725817259028e-07, + "seed": 1468, + "source_expert_id": 254, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022675743103027345, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.6567097902297974, + "relative_to_reference_max": 7.015655849772574e-07, + "seed": 1469, + "source_expert_id": 70, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0023313419580459596, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3703113794326782, + "relative_to_reference_max": 6.089601526924086e-07, + "seed": 1470, + "source_expert_id": 219, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022740622758865357, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4483860731124878, + "relative_to_reference_max": 6.995917594016696e-07, + "seed": 1471, + "source_expert_id": 13, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.0022896772146224977, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4882601499557495, + "relative_to_reference_max": 6.808480098131825e-07, + "seed": 1472, + "source_expert_id": 57, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.00229765202999115, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.046627044677734e-07, + "passed": true, + "reference_max_absolute": 1.105029582977295, + "relative_to_reference_max": 7.281820476694938e-07, + "seed": 1473, + "source_expert_id": 10, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002221005916595459, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.531202793121338, + "relative_to_reference_max": 6.422902591427099e-07, + "seed": 1474, + "source_expert_id": 135, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0023062405586242677, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5133534669876099, + "relative_to_reference_max": 5.907870772652608e-07, + "seed": 1475, + "source_expert_id": 49, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002302670693397522, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.497573971748352, + "relative_to_reference_max": 6.368128282123367e-07, + "seed": 1476, + "source_expert_id": 193, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022995147943496706, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.7468442916870117, + "relative_to_reference_max": 5.971232172971651e-07, + "seed": 1477, + "source_expert_id": 207, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0023493688583374026, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.0505318641662598e-06, + "passed": true, + "reference_max_absolute": 1.6260666847229004, + "relative_to_reference_max": 6.460570615191479e-07, + "seed": 1478, + "source_expert_id": 272, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0023252133369445803, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.5688464641571045, + "relative_to_reference_max": 6.268788318443505e-07, + "seed": 1479, + "source_expert_id": 25, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002313769292831421, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.367616891860962, + "relative_to_reference_max": 6.864299211272409e-07, + "seed": 1480, + "source_expert_id": 45, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022735233783721925, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 7.972121238708496e-07, + "passed": true, + "reference_max_absolute": 1.429183006286621, + "relative_to_reference_max": 5.578096859283321e-07, + "seed": 1481, + "source_expert_id": 101, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002285836601257324, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4457361698150635, + "relative_to_reference_max": 6.59646162500204e-07, + "seed": 1482, + "source_expert_id": 177, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022891472339630126, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3710930347442627, + "relative_to_reference_max": 6.955576990325314e-07, + "seed": 1483, + "source_expert_id": 56, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022742186069488524, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 7.450580596923828e-07, + "passed": true, + "reference_max_absolute": 1.0711196660995483, + "relative_to_reference_max": 6.95588068516649e-07, + "seed": 1484, + "source_expert_id": 29, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022142239332199097, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.5692107677459717, + "relative_to_reference_max": 5.317737068896754e-07, + "seed": 1485, + "source_expert_id": 23, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002313842153549194, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.472805380821228, + "relative_to_reference_max": 6.374047565530672e-07, + "seed": 1486, + "source_expert_id": 119, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022945610761642456, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.4087519645690918, + "relative_to_reference_max": 7.615844612399204e-07, + "seed": 1487, + "source_expert_id": 198, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.0022817503929138186, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.3447247743606567, + "relative_to_reference_max": 6.205470760752596e-07, + "seed": 1488, + "source_expert_id": 35, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002268944954872131, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.8645579814910889, + "relative_to_reference_max": 5.11474744080415e-07, + "seed": 1489, + "source_expert_id": 159, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002372911596298218, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.406358003616333e-07, + "passed": true, + "reference_max_absolute": 1.3102248907089233, + "relative_to_reference_max": 7.179193488323089e-07, + "seed": 1490, + "source_expert_id": 227, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.002262044978141785, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.6544691324234009, + "relative_to_reference_max": 5.764231545434429e-07, + "seed": 1491, + "source_expert_id": 49, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023308938264846803, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.5767594575881958, + "relative_to_reference_max": 6.237328300527633e-07, + "seed": 1492, + "source_expert_id": 43, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023153518915176394, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.699013113975525, + "relative_to_reference_max": 6.665565094330025e-07, + "seed": 1493, + "source_expert_id": 214, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.002339802622795105, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.3796004056930542, + "relative_to_reference_max": 6.264620869033391e-07, + "seed": 1494, + "source_expert_id": 245, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.002275920081138611, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.291815161705017, + "relative_to_reference_max": 7.613137451459328e-07, + "seed": 1495, + "source_expert_id": 127, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0022583630323410033, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.1175870895385742e-06, + "passed": true, + "reference_max_absolute": 1.437887191772461, + "relative_to_reference_max": 7.772425374767697e-07, + "seed": 1496, + "source_expert_id": 248, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0022875774383544923, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0281801223754883e-06, + "passed": true, + "reference_max_absolute": 1.5141428709030151, + "relative_to_reference_max": 6.790509285047157e-07, + "seed": 1497, + "source_expert_id": 150, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.002302828574180603, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4790165424346924, + "relative_to_reference_max": 6.246529146304997e-07, + "seed": 1498, + "source_expert_id": 25, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0022958033084869386, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5477080345153809, + "relative_to_reference_max": 6.546964534553954e-07, + "seed": 1499, + "source_expert_id": 223, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023095416069030764, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.521278977394104, + "relative_to_reference_max": 6.856607493229786e-07, + "seed": 1500, + "source_expert_id": 246, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023042557954788207, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.2095329761505127, + "relative_to_reference_max": 6.899068014757698e-07, + "seed": 1501, + "source_expert_id": 203, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0022419065952301027, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.6192513704299927, + "relative_to_reference_max": 5.797575177985558e-07, + "seed": 1502, + "source_expert_id": 145, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023238502740859984, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.588521957397461, + "relative_to_reference_max": 5.628311698603458e-07, + "seed": 1503, + "source_expert_id": 57, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023177043914794923, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0356307029724121e-06, + "passed": true, + "reference_max_absolute": 1.6081002950668335, + "relative_to_reference_max": 6.440087761624164e-07, + "seed": 1504, + "source_expert_id": 89, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0023216200590133666, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 1.140830159187317, + "relative_to_reference_max": 6.792074839887285e-07, + "seed": 1505, + "source_expert_id": 208, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.0022281660318374634, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.2380008697509766, + "relative_to_reference_max": 6.981153005304521e-07, + "seed": 1506, + "source_expert_id": 134, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.002247600173950195, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.5345350503921509, + "relative_to_reference_max": 7.380009016040488e-07, + "seed": 1507, + "source_expert_id": 196, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0023069070100784302, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4881266355514526, + "relative_to_reference_max": 6.008021429571048e-07, + "seed": 1508, + "source_expert_id": 89, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022976253271102904, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0654330253601074e-06, + "passed": true, + "reference_max_absolute": 1.4490238428115845, + "relative_to_reference_max": 7.352763935842599e-07, + "seed": 1509, + "source_expert_id": 66, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.002289804768562317, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.4178223609924316, + "relative_to_reference_max": 6.621232539704694e-07, + "seed": 1510, + "source_expert_id": 106, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022835644721984866, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.3696364164352417, + "relative_to_reference_max": 6.745381350352236e-07, + "seed": 1511, + "source_expert_id": 205, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022739272832870486, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.1089547872543335, + "relative_to_reference_max": 7.79353098229557e-07, + "seed": 1512, + "source_expert_id": 60, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.002221790957450867, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.6528898477554321, + "relative_to_reference_max": 5.769739088792318e-07, + "seed": 1513, + "source_expert_id": 121, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0023305779695510865, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.7209715843200684, + "relative_to_reference_max": 5.541487872869402e-07, + "seed": 1514, + "source_expert_id": 209, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0023441943168640137, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.642807126045227, + "relative_to_reference_max": 7.075027583537028e-07, + "seed": 1515, + "source_expert_id": 224, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0023285614252090453, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.499977946281433, + "relative_to_reference_max": 6.953977464503521e-07, + "seed": 1516, + "source_expert_id": 286, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022999955892562866, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.1818816661834717, + "relative_to_reference_max": 7.564798551431855e-07, + "seed": 1517, + "source_expert_id": 114, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022363763332366943, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.1380451917648315, + "relative_to_reference_max": 7.332441917894457e-07, + "seed": 1518, + "source_expert_id": 26, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022276090383529664, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.3658623695373535, + "relative_to_reference_max": 8.509573138864713e-07, + "seed": 1519, + "source_expert_id": 248, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.002273172473907471, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.5180883407592773, + "relative_to_reference_max": 5.496814674422097e-07, + "seed": 1520, + "source_expert_id": 188, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0023036176681518554, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.295615315437317, + "relative_to_reference_max": 7.590807449370004e-07, + "seed": 1521, + "source_expert_id": 194, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.0022591230630874632, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5263092517852783, + "relative_to_reference_max": 5.857722939077338e-07, + "seed": 1522, + "source_expert_id": 69, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002305261850357056, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.754432201385498e-07, + "passed": true, + "reference_max_absolute": 1.3696705102920532, + "relative_to_reference_max": 6.391633707232845e-07, + "seed": 1523, + "source_expert_id": 214, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002273934102058411, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1771917343139648e-06, + "passed": true, + "reference_max_absolute": 1.701150894165039, + "relative_to_reference_max": 6.919972463064515e-07, + "seed": 1524, + "source_expert_id": 198, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002340230178833008, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.6793196201324463, + "relative_to_reference_max": 6.03386603142125e-07, + "seed": 1525, + "source_expert_id": 283, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0023358639240264895, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4893581867218018, + "relative_to_reference_max": 6.003053393077855e-07, + "seed": 1526, + "source_expert_id": 272, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0022978716373443604, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.164214134216309e-07, + "passed": true, + "reference_max_absolute": 1.3288477659225464, + "relative_to_reference_max": 6.89636117035129e-07, + "seed": 1527, + "source_expert_id": 56, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0022657695531845094, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.2994358539581299, + "relative_to_reference_max": 6.651096678690015e-07, + "seed": 1528, + "source_expert_id": 89, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002259887170791626, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.6459418535232544, + "relative_to_reference_max": 5.431963891780504e-07, + "seed": 1529, + "source_expert_id": 107, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002329188370704651, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.514708399772644, + "relative_to_reference_max": 7.476609035127863e-07, + "seed": 1530, + "source_expert_id": 60, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0023029416799545286, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.4482115507125854, + "relative_to_reference_max": 8.231483134637677e-07, + "seed": 1531, + "source_expert_id": 50, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0022896423101425173, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.216522216796875, + "relative_to_reference_max": 8.574288814188596e-07, + "seed": 1532, + "source_expert_id": 245, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002243304443359375, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.717179298400879e-07, + "passed": true, + "reference_max_absolute": 1.432356834411621, + "relative_to_reference_max": 6.085899190044842e-07, + "seed": 1533, + "source_expert_id": 61, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0022864713668823244, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.5045617818832397, + "relative_to_reference_max": 6.536631799612381e-07, + "seed": 1534, + "source_expert_id": 185, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002300912356376648, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.3591002225875854, + "relative_to_reference_max": 7.894072770545004e-07, + "seed": 1535, + "source_expert_id": 67, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002271820044517517, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.1650853157043457, + "relative_to_reference_max": 7.162265420459768e-07, + "seed": 1536, + "source_expert_id": 97, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.002233017063140869, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.462237358093262e-07, + "passed": true, + "reference_max_absolute": 1.2170042991638184, + "relative_to_reference_max": 7.775023773206548e-07, + "seed": 1537, + "source_expert_id": 99, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0022434008598327636, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.356290578842163, + "relative_to_reference_max": 7.470957750415517e-07, + "seed": 1538, + "source_expert_id": 93, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022712581157684326, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.274743676185608, + "relative_to_reference_max": 7.948884000065461e-07, + "seed": 1539, + "source_expert_id": 139, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022549487352371217, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.741634130477905e-07, + "passed": true, + "reference_max_absolute": 1.1439622640609741, + "relative_to_reference_max": 8.515695348110424e-07, + "seed": 1540, + "source_expert_id": 36, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022287924528121947, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.7959474325180054, + "relative_to_reference_max": 5.227175017568286e-07, + "seed": 1541, + "source_expert_id": 279, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002359189486503601, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.1175870895385742e-06, + "passed": true, + "reference_max_absolute": 1.4381426572799683, + "relative_to_reference_max": 7.771044714384059e-07, + "seed": 1542, + "source_expert_id": 29, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022876285314559937, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.4612022638320923, + "relative_to_reference_max": 8.158301728752295e-07, + "seed": 1543, + "source_expert_id": 97, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022922404527664186, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.77519690990448, + "relative_to_reference_max": 5.456157974341082e-07, + "seed": 1544, + "source_expert_id": 111, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002355039381980896, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.4129003286361694, + "relative_to_reference_max": 7.593484014492755e-07, + "seed": 1545, + "source_expert_id": 64, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022825800657272338, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.5760788917541504, + "relative_to_reference_max": 6.996387897292723e-07, + "seed": 1546, + "source_expert_id": 0, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.00231521577835083, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.5056411027908325, + "relative_to_reference_max": 7.026790400462129e-07, + "seed": 1547, + "source_expert_id": 195, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0023011282205581667, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.54792320728302, + "relative_to_reference_max": 5.7759304041973e-07, + "seed": 1548, + "source_expert_id": 242, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002309584641456604, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.340294599533081, + "relative_to_reference_max": 6.448338667814144e-07, + "seed": 1549, + "source_expert_id": 24, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002268058919906616, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3624160289764404, + "relative_to_reference_max": 6.99987592719919e-07, + "seed": 1550, + "source_expert_id": 140, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002272483205795288, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.1473894119262695e-06, + "passed": true, + "reference_max_absolute": 1.9157341718673706, + "relative_to_reference_max": 5.989293445696834e-07, + "seed": 1551, + "source_expert_id": 101, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002383146834373474, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.5157359838485718, + "relative_to_reference_max": 7.274920831165603e-07, + "seed": 1552, + "source_expert_id": 8, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0023031471967697144, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4340604543685913, + "relative_to_reference_max": 6.65016815365812e-07, + "seed": 1553, + "source_expert_id": 136, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.0022868120908737183, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.717179298400879e-07, + "passed": true, + "reference_max_absolute": 1.566431999206543, + "relative_to_reference_max": 5.564990566342146e-07, + "seed": 1554, + "source_expert_id": 147, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002313286399841309, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.791685104370117e-07, + "passed": true, + "reference_max_absolute": 1.4283517599105835, + "relative_to_reference_max": 6.15512603486447e-07, + "seed": 1555, + "source_expert_id": 241, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0022856703519821167, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.489781379699707, + "relative_to_reference_max": 8.401887399985533e-07, + "seed": 1556, + "source_expert_id": 11, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0022979562759399413, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.2218952178955078e-06, + "passed": true, + "reference_max_absolute": 1.42347252368927, + "relative_to_reference_max": 8.58390448400559e-07, + "seed": 1557, + "source_expert_id": 251, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0022846945047378543, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.6206340789794922, + "relative_to_reference_max": 5.149003329492667e-07, + "seed": 1558, + "source_expert_id": 178, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0023241268157958984, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.611631155014038, + "relative_to_reference_max": 6.287288242282798e-07, + "seed": 1559, + "source_expert_id": 268, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0023223262310028076, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.259771466255188, + "relative_to_reference_max": 7.806786112702171e-07, + "seed": 1560, + "source_expert_id": 118, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0022519542932510376, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.5821064710617065, + "relative_to_reference_max": 6.969732748800057e-07, + "seed": 1561, + "source_expert_id": 154, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0023164212942123415, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.458860993385315, + "relative_to_reference_max": 7.354255208835108e-07, + "seed": 1562, + "source_expert_id": 103, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002291772198677063, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.4862922430038452, + "relative_to_reference_max": 8.02058209695392e-07, + "seed": 1563, + "source_expert_id": 248, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0022972584486007693, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.3209470510482788, + "relative_to_reference_max": 7.670852214534424e-07, + "seed": 1564, + "source_expert_id": 25, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002264189410209656, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.908900260925293, + "relative_to_reference_max": 5.932673769888196e-07, + "seed": 1565, + "source_expert_id": 157, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002381780052185059, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.275913953781128, + "relative_to_reference_max": 8.175169496957061e-07, + "seed": 1566, + "source_expert_id": 197, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002255182790756226, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.502367377281189, + "relative_to_reference_max": 6.34781033472718e-07, + "seed": 1567, + "source_expert_id": 66, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0023004734754562377, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 2.013488292694092, + "relative_to_reference_max": 5.328481967588147e-07, + "seed": 1568, + "source_expert_id": 210, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0024026976585388186, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.4571417570114136, + "relative_to_reference_max": 6.647091629483177e-07, + "seed": 1569, + "source_expert_id": 98, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.002291428351402283, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.7150613069534302, + "relative_to_reference_max": 5.90812093464805e-07, + "seed": 1570, + "source_expert_id": 206, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0023430122613906862, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.489533543586731, + "relative_to_reference_max": 6.002346677457018e-07, + "seed": 1571, + "source_expert_id": 72, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022979067087173463, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.5144144296646118, + "relative_to_reference_max": 7.871642478814851e-07, + "seed": 1572, + "source_expert_id": 170, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0023028828859329225, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.6677660942077637, + "relative_to_reference_max": 6.075665914427748e-07, + "seed": 1573, + "source_expert_id": 171, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002333553218841553, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.473956823348999, + "relative_to_reference_max": 6.470164534666575e-07, + "seed": 1574, + "source_expert_id": 237, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022947913646698, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.7463918924331665, + "relative_to_reference_max": 5.460826522033036e-07, + "seed": 1575, + "source_expert_id": 8, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0023492783784866333, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.2671830654144287, + "relative_to_reference_max": 7.525939561813458e-07, + "seed": 1576, + "source_expert_id": 168, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002253436613082886, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.4158178567886353, + "relative_to_reference_max": 6.735854558081211e-07, + "seed": 1577, + "source_expert_id": 61, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002283163571357727, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.30597722530365, + "relative_to_reference_max": 6.389583299673922e-07, + "seed": 1578, + "source_expert_id": 172, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.00226119544506073, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.4918618202209473, + "relative_to_reference_max": 6.592277015630648e-07, + "seed": 1579, + "source_expert_id": 202, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022983723640441897, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5604668855667114, + "relative_to_reference_max": 6.493434564704974e-07, + "seed": 1580, + "source_expert_id": 130, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002312093377113342, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3601031303405762, + "relative_to_reference_max": 7.011779438868327e-07, + "seed": 1581, + "source_expert_id": 222, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022720206260681153, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.696452260017395, + "relative_to_reference_max": 6.675626997725298e-07, + "seed": 1582, + "source_expert_id": 256, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002339290452003479, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0356307029724121e-06, + "passed": true, + "reference_max_absolute": 1.3468294143676758, + "relative_to_reference_max": 7.689397721229836e-07, + "seed": 1583, + "source_expert_id": 128, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022693658828735353, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.3729751110076904, + "relative_to_reference_max": 6.728978454245117e-07, + "seed": 1584, + "source_expert_id": 226, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.0022745950222015383, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3702497482299805, + "relative_to_reference_max": 6.524866527330317e-07, + "seed": 1585, + "source_expert_id": 252, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002274049949645996, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 8.121132850646973e-07, + "passed": true, + "reference_max_absolute": 1.3403394222259521, + "relative_to_reference_max": 6.059012154667436e-07, + "seed": 1586, + "source_expert_id": 52, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022680678844451906, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4596959352493286, + "relative_to_reference_max": 6.941712562956228e-07, + "seed": 1587, + "source_expert_id": 258, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002291939187049866, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.642673492431641e-07, + "passed": true, + "reference_max_absolute": 1.3363152742385864, + "relative_to_reference_max": 6.467540751082198e-07, + "seed": 1588, + "source_expert_id": 18, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022672630548477173, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.6027295589447021, + "relative_to_reference_max": 7.437891744460816e-07, + "seed": 1589, + "source_expert_id": 59, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0023205459117889406, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.996576070785522e-07, + "passed": true, + "reference_max_absolute": 1.2435739040374756, + "relative_to_reference_max": 7.234452284320697e-07, + "seed": 1590, + "source_expert_id": 80, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022487147808074952, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.5518743991851807, + "relative_to_reference_max": 7.489591771926798e-07, + "seed": 1591, + "source_expert_id": 172, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002310374879837036, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 9.387731552124023e-07, + "passed": true, + "reference_max_absolute": 1.457487940788269, + "relative_to_reference_max": 6.441035489492115e-07, + "seed": 1592, + "source_expert_id": 137, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022914975881576536, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.296401023864746e-06, + "passed": true, + "reference_max_absolute": 1.256840705871582, + "relative_to_reference_max": 1.0314759999484025e-06, + "seed": 1593, + "source_expert_id": 40, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022513681411743163, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.6150976419448853, + "relative_to_reference_max": 6.458317172163458e-07, + "seed": 1594, + "source_expert_id": 197, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002323019528388977, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3745671510696411, + "relative_to_reference_max": 7.154809701575055e-07, + "seed": 1595, + "source_expert_id": 89, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022749134302139282, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.4761879444122314, + "relative_to_reference_max": 7.16699014355138e-07, + "seed": 1596, + "source_expert_id": 35, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002295237588882446, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.528383731842041, + "relative_to_reference_max": 5.459787417717103e-07, + "seed": 1597, + "source_expert_id": 154, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0023056767463684085, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4004578590393066, + "relative_to_reference_max": 6.596928197841792e-07, + "seed": 1598, + "source_expert_id": 84, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002280091571807861, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.080334186553955e-06, + "passed": true, + "reference_max_absolute": 1.3166143894195557, + "relative_to_reference_max": 8.205395560276632e-07, + "seed": 1599, + "source_expert_id": 19, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0022633228778839113, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.5740402936935425, + "relative_to_reference_max": 6.816112714875114e-07, + "seed": 1600, + "source_expert_id": 255, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0023148080587387086, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.087784767150879e-06, + "passed": true, + "reference_max_absolute": 1.9297642707824707, + "relative_to_reference_max": 5.636878988902668e-07, + "seed": 1601, + "source_expert_id": 38, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.002385952854156494, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3870270252227783, + "relative_to_reference_max": 7.090536960777556e-07, + "seed": 1602, + "source_expert_id": 220, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0022774054050445558, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.3776593208312988, + "relative_to_reference_max": 8.653031104877852e-07, + "seed": 1603, + "source_expert_id": 31, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0022755318641662597, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.601271152496338, + "relative_to_reference_max": 5.95573282463434e-07, + "seed": 1604, + "source_expert_id": 280, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0023202542304992676, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4814261198043823, + "relative_to_reference_max": 6.035195813537556e-07, + "seed": 1605, + "source_expert_id": 106, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0022962852239608767, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 8.381903171539307e-07, + "passed": true, + "reference_max_absolute": 1.348297357559204, + "relative_to_reference_max": 6.216657716153134e-07, + "seed": 1606, + "source_expert_id": 10, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002269659471511841, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.2036478519439697, + "relative_to_reference_max": 6.93280036605186e-07, + "seed": 1607, + "source_expert_id": 178, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002240729570388794, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5470857620239258, + "relative_to_reference_max": 5.779056944207289e-07, + "seed": 1608, + "source_expert_id": 234, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002309417152404785, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 8.903443813323975e-07, + "passed": true, + "reference_max_absolute": 1.373024821281433, + "relative_to_reference_max": 6.484546874407166e-07, + "seed": 1609, + "source_expert_id": 36, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0022746049642562867, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.3467872142791748, + "relative_to_reference_max": 7.744959801445784e-07, + "seed": 1610, + "source_expert_id": 7, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002269357442855835, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5075397491455078, + "relative_to_reference_max": 6.721407921455999e-07, + "seed": 1611, + "source_expert_id": 212, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0023015079498291016, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.313225746154785e-07, + "passed": true, + "reference_max_absolute": 1.260894775390625, + "relative_to_reference_max": 7.386203772055086e-07, + "seed": 1612, + "source_expert_id": 187, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002252178955078125, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.2405216693878174e-06, + "passed": true, + "reference_max_absolute": 1.3578298091888428, + "relative_to_reference_max": 9.136061537262138e-07, + "seed": 1613, + "source_expert_id": 199, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002271565961837769, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.5128700733184814, + "relative_to_reference_max": 5.515774563674653e-07, + "seed": 1614, + "source_expert_id": 221, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0023025740146636964, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.760260581970215e-07, + "passed": true, + "reference_max_absolute": 1.553640365600586, + "relative_to_reference_max": 6.282187820343624e-07, + "seed": 1615, + "source_expert_id": 47, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0023107280731201173, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0617077350616455e-06, + "passed": true, + "reference_max_absolute": 1.45469069480896, + "relative_to_reference_max": 7.298511902566863e-07, + "seed": 1616, + "source_expert_id": 186, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.002290938138961792, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.4987125396728516, + "relative_to_reference_max": 6.164437606028327e-07, + "seed": 1617, + "source_expert_id": 147, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0022997425079345703, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.4130845069885254, + "relative_to_reference_max": 7.592494296349562e-07, + "seed": 1618, + "source_expert_id": 36, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002282616901397705, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.2234989404678345, + "relative_to_reference_max": 6.820316710175415e-07, + "seed": 1619, + "source_expert_id": 197, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002244699788093567, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.2507884502410889, + "relative_to_reference_max": 7.148048668490086e-07, + "seed": 1620, + "source_expert_id": 33, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002250157690048218, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.6285761594772339, + "relative_to_reference_max": 5.12389317502553e-07, + "seed": 1621, + "source_expert_id": 93, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0023257152318954468, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 9.685754776000977e-07, + "passed": true, + "reference_max_absolute": 1.5798721313476562, + "relative_to_reference_max": 6.130720698097809e-07, + "seed": 1622, + "source_expert_id": 181, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002315974426269531, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.1799885034561157, + "relative_to_reference_max": 7.576935444813087e-07, + "seed": 1623, + "source_expert_id": 134, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0022359977006912233, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4130293130874634, + "relative_to_reference_max": 7.170969149731439e-07, + "seed": 1624, + "source_expert_id": 84, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0022826058626174927, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.3585659265518188, + "relative_to_reference_max": 6.690664141207849e-07, + "seed": 1625, + "source_expert_id": 244, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0022717131853103637, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.3319838047027588, + "relative_to_reference_max": 7.159804143557653e-07, + "seed": 1626, + "source_expert_id": 264, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002266396760940552, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5273038148880005, + "relative_to_reference_max": 6.634429583062005e-07, + "seed": 1627, + "source_expert_id": 164, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0023054607629776, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.4874283075332642, + "relative_to_reference_max": 7.012649136005562e-07, + "seed": 1628, + "source_expert_id": 207, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002297485661506653, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.8940962553024292, + "relative_to_reference_max": 5.664356301605826e-07, + "seed": 1629, + "source_expert_id": 202, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0023788192510604857, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.2218952178955078e-06, + "passed": true, + "reference_max_absolute": 1.5885215997695923, + "relative_to_reference_max": 7.692027719816577e-07, + "seed": 1630, + "source_expert_id": 71, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0023177043199539187, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.08970832824707e-07, + "passed": true, + "reference_max_absolute": 1.3451581001281738, + "relative_to_reference_max": 6.75735315230303e-07, + "seed": 1631, + "source_expert_id": 59, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002269031620025635, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.7902145385742188, + "relative_to_reference_max": 5.327150997029581e-07, + "seed": 1632, + "source_expert_id": 214, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0023580429077148437, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.1473894119262695e-06, + "passed": true, + "reference_max_absolute": 1.5889959335327148, + "relative_to_reference_max": 7.220845489361012e-07, + "seed": 1633, + "source_expert_id": 284, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.002317799186706543, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 9.294599294662476e-07, + "passed": true, + "reference_max_absolute": 1.3093843460083008, + "relative_to_reference_max": 7.098449987581838e-07, + "seed": 1634, + "source_expert_id": 54, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.00226187686920166, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.5595934391021729, + "relative_to_reference_max": 6.114890538108839e-07, + "seed": 1635, + "source_expert_id": 143, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0023119186878204348, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.4045919179916382, + "relative_to_reference_max": 8.274934222760041e-07, + "seed": 1636, + "source_expert_id": 130, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.002280918383598328, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0579824447631836e-06, + "passed": true, + "reference_max_absolute": 1.4440925121307373, + "relative_to_reference_max": 7.326278862855857e-07, + "seed": 1637, + "source_expert_id": 15, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0022888185024261473, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.4407602548599243, + "relative_to_reference_max": 7.032946375107738e-07, + "seed": 1638, + "source_expert_id": 14, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0022881520509719848, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 2.040210723876953, + "relative_to_reference_max": 6.135138520910422e-07, + "seed": 1639, + "source_expert_id": 107, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0024080421447753905, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.651252269744873, + "relative_to_reference_max": 7.219326309795153e-07, + "seed": 1640, + "source_expert_id": 76, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0023302504539489746, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.3824774026870728, + "relative_to_reference_max": 7.113871350681004e-07, + "seed": 1641, + "source_expert_id": 225, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0022764954805374145, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5090025663375854, + "relative_to_reference_max": 6.714892232694557e-07, + "seed": 1642, + "source_expert_id": 8, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.002301800513267517, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.648501873016357e-07, + "passed": true, + "reference_max_absolute": 1.3701387643814087, + "relative_to_reference_max": 7.041988829045699e-07, + "seed": 1643, + "source_expert_id": 195, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.002274027752876282, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5620198249816895, + "relative_to_reference_max": 5.72380489243368e-07, + "seed": 1644, + "source_expert_id": 281, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.002312403964996338, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.197930097579956, + "relative_to_reference_max": 7.712236263910138e-07, + "seed": 1645, + "source_expert_id": 212, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0022395860195159913, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.2801814079284668, + "relative_to_reference_max": 7.216727163016089e-07, + "seed": 1646, + "source_expert_id": 161, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0022560362815856935, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 9.611248970031738e-07, + "passed": true, + "reference_max_absolute": 1.5795104503631592, + "relative_to_reference_max": 6.084954340012079e-07, + "seed": 1647, + "source_expert_id": 177, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.002315902090072632, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.5313560962677002, + "relative_to_reference_max": 6.616873525702194e-07, + "seed": 1648, + "source_expert_id": 258, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.00230627121925354, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.1101365089416504e-06, + "passed": true, + "reference_max_absolute": 1.7384026050567627, + "relative_to_reference_max": 6.385957462974476e-07, + "seed": 1649, + "source_expert_id": 259, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0023476805210113525, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.4901161193847656e-06, + "passed": true, + "reference_max_absolute": 1.4700613021850586, + "relative_to_reference_max": 1.013642163881124e-06, + "seed": 1650, + "source_expert_id": 5, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022940122604370116, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.2656067609786987, + "relative_to_reference_max": 7.064356000591145e-07, + "seed": 1651, + "source_expert_id": 187, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.00225312135219574, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.349643349647522, + "relative_to_reference_max": 9.27428376252216e-07, + "seed": 1652, + "source_expert_id": 78, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022699286699295043, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.495530366897583, + "relative_to_reference_max": 6.576106113004764e-07, + "seed": 1653, + "source_expert_id": 105, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022991060733795164, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.3392189741134644, + "relative_to_reference_max": 7.788728383719581e-07, + "seed": 1654, + "source_expert_id": 165, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022678437948226928, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.6683906316757202, + "relative_to_reference_max": 6.787908234625835e-07, + "seed": 1655, + "source_expert_id": 9, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0023336781263351442, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.376338243484497, + "relative_to_reference_max": 7.578669622145722e-07, + "seed": 1656, + "source_expert_id": 156, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022752676486968992, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.2149730920791626, + "relative_to_reference_max": 8.094637199832455e-07, + "seed": 1657, + "source_expert_id": 139, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022429946184158328, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.043081283569336e-06, + "passed": true, + "reference_max_absolute": 1.5438356399536133, + "relative_to_reference_max": 6.756427022248799e-07, + "seed": 1658, + "source_expert_id": 235, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.002308767127990723, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.5790252685546875, + "relative_to_reference_max": 6.794594281186281e-07, + "seed": 1659, + "source_expert_id": 121, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0023158050537109374, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.4012213945388794, + "relative_to_reference_max": 7.018709838623271e-07, + "seed": 1660, + "source_expert_id": 62, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.002280244278907776, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 9.238719940185547e-07, + "passed": true, + "reference_max_absolute": 1.303887963294983, + "relative_to_reference_max": 7.085516701019995e-07, + "seed": 1661, + "source_expert_id": 256, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.002260777592658997, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 1.4901056289672852, + "relative_to_reference_max": 5.600039424277546e-07, + "seed": 1662, + "source_expert_id": 14, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.002298021125793457, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.4148460626602173, + "relative_to_reference_max": 6.31920104403312e-07, + "seed": 1663, + "source_expert_id": 201, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022829692125320433, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.3996201753616333, + "relative_to_reference_max": 7.23967101231468e-07, + "seed": 1664, + "source_expert_id": 180, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0022799240350723266, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.3377444744110107, + "relative_to_reference_max": 6.683411434194151e-07, + "seed": 1665, + "source_expert_id": 277, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.002267548894882202, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.3922771215438843, + "relative_to_reference_max": 8.348126641851753e-07, + "seed": 1666, + "source_expert_id": 144, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002278455424308777, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.5515815019607544, + "relative_to_reference_max": 7.683082674041606e-07, + "seed": 1667, + "source_expert_id": 154, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002310316300392151, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.3904825448989868, + "relative_to_reference_max": 7.715908480066336e-07, + "seed": 1668, + "source_expert_id": 125, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0022780965089797974, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.3465423583984375, + "relative_to_reference_max": 7.967692952734937e-07, + "seed": 1669, + "source_expert_id": 98, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0022693084716796878, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.5008844137191772, + "relative_to_reference_max": 8.339733085651203e-07, + "seed": 1670, + "source_expert_id": 92, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0023001768827438356, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.312949538230896, + "relative_to_reference_max": 6.809627069411619e-07, + "seed": 1671, + "source_expert_id": 123, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0022625899076461793, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.7891242504119873, + "relative_to_reference_max": 6.662996688090867e-07, + "seed": 1672, + "source_expert_id": 196, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0023578248500823974, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.430511474609375e-06, + "passed": true, + "reference_max_absolute": 2.2389421463012695, + "relative_to_reference_max": 6.389229292827324e-07, + "seed": 1673, + "source_expert_id": 80, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002447788429260254, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5670093297958374, + "relative_to_reference_max": 5.70557976031544e-07, + "seed": 1674, + "source_expert_id": 141, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0023134018659591677, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 1.2480577230453491, + "relative_to_reference_max": 7.641267697772962e-07, + "seed": 1675, + "source_expert_id": 83, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.00224961154460907, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 9.98377799987793e-07, + "passed": true, + "reference_max_absolute": 1.7211545705795288, + "relative_to_reference_max": 5.800628351767557e-07, + "seed": 1676, + "source_expert_id": 249, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002344230914115906, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.8157646656036377, + "relative_to_reference_max": 5.416322155751366e-07, + "seed": 1677, + "source_expert_id": 250, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0023631529331207277, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.8155252933502197, + "relative_to_reference_max": 5.58118889829301e-07, + "seed": 1678, + "source_expert_id": 20, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002363105058670044, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.3180348873138428, + "relative_to_reference_max": 7.687800762593658e-07, + "seed": 1679, + "source_expert_id": 34, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0022636069774627687, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 9.834766387939453e-07, + "passed": true, + "reference_max_absolute": 1.639801025390625, + "relative_to_reference_max": 5.997536430126738e-07, + "seed": 1680, + "source_expert_id": 134, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.002327960205078125, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 8.940696716308594e-07, + "passed": true, + "reference_max_absolute": 1.5556824207305908, + "relative_to_reference_max": 5.74712203285668e-07, + "seed": 1681, + "source_expert_id": 184, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0023111364841461183, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 2.1151535511016846, + "relative_to_reference_max": 5.917762044421089e-07, + "seed": 1682, + "source_expert_id": 249, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002423030710220337, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.430511474609375e-06, + "passed": true, + "reference_max_absolute": 1.9862048625946045, + "relative_to_reference_max": 7.202235285743284e-07, + "seed": 1683, + "source_expert_id": 148, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002397240972518921, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.6340211629867554, + "relative_to_reference_max": 8.025001234755872e-07, + "seed": 1684, + "source_expert_id": 151, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002326804232597351, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.3901658058166504, + "relative_to_reference_max": 7.717666493219258e-07, + "seed": 1685, + "source_expert_id": 221, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.00227803316116333, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 1.440638780593872, + "relative_to_reference_max": 7.033539391213239e-07, + "seed": 1686, + "source_expert_id": 18, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0022881277561187744, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 8.046627044677734e-07, + "passed": true, + "reference_max_absolute": 1.152589201927185, + "relative_to_reference_max": 6.981348629002756e-07, + "seed": 1687, + "source_expert_id": 87, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002230517840385437, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.1026859283447266e-06, + "passed": true, + "reference_max_absolute": 1.7064692974090576, + "relative_to_reference_max": 6.461797642764163e-07, + "seed": 1688, + "source_expert_id": 239, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0023412938594818116, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.0915100574493408e-06, + "passed": true, + "reference_max_absolute": 1.4409856796264648, + "relative_to_reference_max": 7.574746042807893e-07, + "seed": 1689, + "source_expert_id": 105, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0022881971359252932, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.1622905731201172e-06, + "passed": true, + "reference_max_absolute": 1.8609517812728882, + "relative_to_reference_max": 6.245678070847769e-07, + "seed": 1690, + "source_expert_id": 207, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0023721903562545776, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.9069628715515137, + "relative_to_reference_max": 5.938701102297939e-07, + "seed": 1691, + "source_expert_id": 35, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002381392574310303, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.3709068298339844e-06, + "passed": true, + "reference_max_absolute": 1.9819679260253906, + "relative_to_reference_max": 6.916897149709082e-07, + "seed": 1692, + "source_expert_id": 147, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002396393585205078, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 1.0728836059570312e-06, + "passed": true, + "reference_max_absolute": 1.5516501665115356, + "relative_to_reference_max": 6.91446840990659e-07, + "seed": 1693, + "source_expert_id": 20, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0023103300333023073, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.7918777465820312, + "relative_to_reference_max": 6.652757967342942e-07, + "seed": 1694, + "source_expert_id": 84, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0023583755493164064, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.5927348136901855, + "relative_to_reference_max": 7.11033777247937e-07, + "seed": 1695, + "source_expert_id": 130, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002318546962738037, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.7591885328292847, + "relative_to_reference_max": 6.7763794116517e-07, + "seed": 1696, + "source_expert_id": 52, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.002351837706565857, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 1.3709068298339844e-06, + "passed": true, + "reference_max_absolute": 1.891948938369751, + "relative_to_reference_max": 7.246003325096413e-07, + "seed": 1697, + "source_expert_id": 17, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0023783897876739502, + "tokens": 2 + }, + { + "expert_slot": 0, + "max_absolute_error": 1.3709068298339844e-06, + "passed": true, + "reference_max_absolute": 2.1097512245178223, + "relative_to_reference_max": 6.497954895830438e-07, + "seed": 1698, + "source_expert_id": 44, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0024219502449035644, + "tokens": 2 + }, + { + "expert_slot": 1, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.717604398727417, + "relative_to_reference_max": 7.28746119426914e-07, + "seed": 1699, + "source_expert_id": 212, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023435208797454835, + "tokens": 2 + }, + { + "expert_slot": 2, + "max_absolute_error": 1.2069940567016602e-06, + "passed": true, + "reference_max_absolute": 1.8899418115615845, + "relative_to_reference_max": 6.38640856198831e-07, + "seed": 1700, + "source_expert_id": 152, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.002377988362312317, + "tokens": 2 + }, + { + "expert_slot": 3, + "max_absolute_error": 1.1920928955078125e-06, + "passed": true, + "reference_max_absolute": 1.7604910135269165, + "relative_to_reference_max": 6.771365978856138e-07, + "seed": 1701, + "source_expert_id": 127, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023520982027053832, + "tokens": 2 + }, + { + "expert_slot": 4, + "max_absolute_error": 1.2218952178955078e-06, + "passed": true, + "reference_max_absolute": 1.9688467979431152, + "relative_to_reference_max": 6.20614676150547e-07, + "seed": 1702, + "source_expert_id": 244, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023937693595886232, + "tokens": 2 + }, + { + "expert_slot": 5, + "max_absolute_error": 1.1324882507324219e-06, + "passed": true, + "reference_max_absolute": 1.6444451808929443, + "relative_to_reference_max": 6.886749791911418e-07, + "seed": 1703, + "source_expert_id": 138, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.002328889036178589, + "tokens": 2 + }, + { + "expert_slot": 6, + "max_absolute_error": 1.3709068298339844e-06, + "passed": true, + "reference_max_absolute": 2.0499982833862305, + "relative_to_reference_max": 6.687355989242545e-07, + "seed": 1704, + "source_expert_id": 156, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0024099996566772463, + "tokens": 2 + }, + { + "expert_slot": 7, + "max_absolute_error": 1.3113021850585938e-06, + "passed": true, + "reference_max_absolute": 1.8506451845169067, + "relative_to_reference_max": 7.085648810638419e-07, + "seed": 1705, + "source_expert_id": 266, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023701290369033813, + "tokens": 2 + }, + { + "expert_slot": 8, + "max_absolute_error": 1.2218952178955078e-06, + "passed": true, + "reference_max_absolute": 1.7663606405258179, + "relative_to_reference_max": 6.917586306337583e-07, + "seed": 1706, + "source_expert_id": 24, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023532721281051636, + "tokens": 2 + }, + { + "expert_slot": 9, + "max_absolute_error": 1.0468065738677979e-06, + "passed": true, + "reference_max_absolute": 1.5995798110961914, + "relative_to_reference_max": 6.544259727499447e-07, + "seed": 1707, + "source_expert_id": 135, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023199159622192384, + "tokens": 2 + }, + { + "expert_slot": 10, + "max_absolute_error": 1.6540288925170898e-06, + "passed": true, + "reference_max_absolute": 2.138136148452759, + "relative_to_reference_max": 7.735844575257808e-07, + "seed": 1708, + "source_expert_id": 113, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0024276272296905516, + "tokens": 2 + }, + { + "expert_slot": 11, + "max_absolute_error": 3.5762786865234375e-06, + "passed": true, + "reference_max_absolute": 3.330444097518921, + "relative_to_reference_max": 1.0738143568263632e-06, + "seed": 1709, + "source_expert_id": 185, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0026660888195037843, + "tokens": 2 + }, + { + "expert_slot": 12, + "max_absolute_error": 1.4677643775939941e-06, + "passed": true, + "reference_max_absolute": 1.8904376029968262, + "relative_to_reference_max": 7.764151407416002e-07, + "seed": 1710, + "source_expert_id": 147, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023780875205993653, + "tokens": 2 + }, + { + "expert_slot": 13, + "max_absolute_error": 1.6689300537109375e-06, + "passed": true, + "reference_max_absolute": 2.6332759857177734, + "relative_to_reference_max": 6.337847087668723e-07, + "seed": 1711, + "source_expert_id": 186, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.002526655197143555, + "tokens": 2 + }, + { + "expert_slot": 14, + "max_absolute_error": 1.2516975402832031e-06, + "passed": true, + "reference_max_absolute": 1.9712671041488647, + "relative_to_reference_max": 6.349710486462206e-07, + "seed": 1712, + "source_expert_id": 17, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.0023942534208297728, + "tokens": 2 + }, + { + "expert_slot": 15, + "max_absolute_error": 2.1457672119140625e-06, + "passed": true, + "reference_max_absolute": 4.514376640319824, + "relative_to_reference_max": 4.753186060616431e-07, + "seed": 1713, + "source_expert_id": 184, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.002902875328063965, + "tokens": 2 + } + ], + "weight_block_size": [ + 128, + 128 + ] + }, + "output": { + "directory": "\\artifacts\\folded-glm-experts-fixed16-top4-identity-hadamard.staging-20260827T204902", + "index": { + "name": "model.safetensors.index.json", + "sha256": "c156f43793b0cee11290e32619c767d86eaaf4ca1a6ec268ece9eb8d62b6880c", + "size": 147536 + }, + "loader_api": "load_folded_expert(layer_index, expert_index, gate_proj=..., up_proj=..., down_proj=...)", + "router_tensor_key_template": "model.layers.{target}.feed_forward.router.{proj.weight|e_score_correction_bias}", + "shards": [ + { + "name": "folded-experts-00001.safetensors", + "sha256": "7a3c00d82bd32c368119d3e5f16b39853c90469896e538826e0002012421d328", + "size": 1065764144 + }, + { + "name": "folded-experts-00002.safetensors", + "sha256": "5e4fb146615bf0c3bfccabe0ae38ab8cb0d363ef7847f217fcefdfd23f6664a4", + "size": 1065764160 + }, + { + "name": "folded-experts-00003.safetensors", + "sha256": "0c6dbfc8992071257c8df92f0b36a2ab98f140075ac6e40c9f4d833ae311e676", + "size": 1065632784 + }, + { + "name": "folded-experts-00004.safetensors", + "sha256": "bb4f125e99943f1311001b445814ab175ec6d84834ac2f0db2e5a58643930ebb", + "size": 1065764176 + }, + { + "name": "folded-experts-00005.safetensors", + "sha256": "728b595ad607555f988111f81c81a9dd834ce547789156882ac9242aee9c2bdb", + "size": 1065764296 + }, + { + "name": "folded-experts-00006.safetensors", + "sha256": "a4569393d7bc1aa460b73c3af0a69e4ef07f753631c891b3d0e94480732e4883", + "size": 1065632912 + }, + { + "name": "folded-experts-00007.safetensors", + "sha256": "e87ca74e4b915cf64a1ca828e3c6ee61d48d5f8978eafc0c99b341e52913c129", + "size": 1065764280 + }, + { + "name": "folded-experts-00008.safetensors", + "sha256": "75d3b4f25ade944ab388e400916dd6c0b985e917a6be70cc4248e19bfd540c7c", + "size": 1065764296 + }, + { + "name": "folded-experts-00009.safetensors", + "sha256": "1661c4c503ff76d696f1365a7278c11347275e1b351dc05a422d68e5349df0e4", + "size": 1065632912 + }, + { + "name": "folded-experts-00010.safetensors", + "sha256": "b801424e1a7837de67dfb53ce988e9affb0df60fae8b4e4e3053a44bd67f6055", + "size": 1065764288 + }, + { + "name": "folded-experts-00011.safetensors", + "sha256": "bfab6135cb28633ab4c37f192bdf73593350c393e55a896bdf41be19b7615ce3", + "size": 1065764296 + }, + { + "name": "folded-experts-00012.safetensors", + "sha256": "4b645673ecf919004d968008eccf44551e6510b33d820003c1c0be22249c8f8d", + "size": 360716032 + } + ], + "tensor_count": 1500, + "tensor_key_template": "model.layers.{target}.feed_forward.experts.{slot}.{gate_proj|up_proj|down_proj}.weight", + "tensor_storage_bytes": 12083529600 + }, + "projections": { + "bias_free_validated": true, + "file_sha256": "497cae6c91cc46fe16d4c78a95746eb4bac83d140a5daca5658a053b7b0db729", + "path": "\\artifacts\\fixed-bridges\\lfm2048-glm4096-identity-hadamard.safetensors", + "tensor_fingerprints": { + "P_in": { + "dtype": "torch.float32", + "numel": 8388608, + "sha256": "477152e3bcca9a407c222443c0f1e832a21cc44ae500ab35d86a4580293a3cbd", + "shape": [ + 4096, + 2048 + ], + "storage_bytes": 33554432 + }, + "P_out": { + "dtype": "torch.float32", + "numel": 8388608, + "sha256": "26c5f05e8f90d54f5d5b2e8fa91d7a846aaf3cde92c66eac79da6e351e27b408", + "shape": [ + 2048, + 4096 + ], + "storage_bytes": 33554432 + } + }, + "tied_bridge_required": true, + "tied_bridge_validation": { + "P_in|P_out": { + "gram_atol": 1e-05, + "gram_max_absolute_error": 5.960464477539063e-08, + "p_in_key": "P_in", + "p_out_key": "P_out", + "transpose_exact": true + } + } + }, + "router_initialization": { + "correction_output_dtype": "float32", + "correction_semantics": "e_score_correction_bias is added after sigmoid for top-k choice only; route weights use uncorrected sigmoid scores", + "equation": "R_glm[selected expert rows] @ P_in", + "layers": [ + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.0.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "3337604ec64d342e414c09f72a7ad955523fecab5eede1f91394d3d3b9db4d29", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.0.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "1112f1374ca4d9c694137fbbd70fbd4b7762d4b16e2f53b9c0d37ac1228e6fbc", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "3337604ec64d342e414c09f72a7ad955523fecab5eede1f91394d3d3b9db4d29", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "24f04c209b0c974c9346d5cc923ca05977b3360251117638d1ff6feb9f9c6c9f", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 149, + 171, + 186, + 98, + 286, + 189, + 12, + 10, + 202, + 177, + 66, + 24, + 118, + 107, + 220, + 47 + ], + "source_correction": { + "name": "model.language_model.layers.3.mlp.gate.e_score_correction_bias", + "shard": "model-00032-of-00062.safetensors" + }, + "source_layer": 3, + "source_weight": { + "name": "model.language_model.layers.3.mlp.gate.weight", + "shard": "model-00032-of-00062.safetensors" + }, + "target_layer": 0 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.1.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "3193cfc3a0811641f2c4e1c2862f52bb870c0ffe987bb66f67411d4a4c125d7d", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.1.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "31e66b64223afdcb666c5d2f873948c91a9a4625fbd558253c2041627be8bd1d", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "3193cfc3a0811641f2c4e1c2862f52bb870c0ffe987bb66f67411d4a4c125d7d", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "480c568d42b6f28cf15fab6d1b68cfba5370639f4e9b3242f964e723bd02fc9f", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 241, + 243, + 173, + 268, + 39, + 45, + 216, + 197, + 205, + 223, + 89, + 178, + 248, + 93, + 142, + 74 + ], + "source_correction": { + "name": "model.language_model.layers.4.mlp.gate.e_score_correction_bias", + "shard": "model-00047-of-00062.safetensors" + }, + "source_layer": 4, + "source_weight": { + "name": "model.language_model.layers.4.mlp.gate.weight", + "shard": "model-00047-of-00062.safetensors" + }, + "target_layer": 1 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.2.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "48eba7c71062b898c548164a6fae63bebbdfacb82ddffaf61775c1c63349b071", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.2.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "69aaef2d03a7bd5b23f04b8cb78d05d1326078157b8943b3f150283dd1f77c87", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "48eba7c71062b898c548164a6fae63bebbdfacb82ddffaf61775c1c63349b071", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "bea78e03f0cf1a70628f0f5742ad73d06b7b08c0fce47031fea5c6c0a0b74f1b", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 160, + 276, + 104, + 234, + 16, + 83, + 209, + 260, + 270, + 282, + 55, + 174, + 236, + 256, + 137, + 206 + ], + "source_correction": { + "name": "model.language_model.layers.6.mlp.gate.e_score_correction_bias", + "shard": "model-00057-of-00062.safetensors" + }, + "source_layer": 6, + "source_weight": { + "name": "model.language_model.layers.6.mlp.gate.weight", + "shard": "model-00057-of-00062.safetensors" + }, + "target_layer": 2 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.3.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "e4123fdce0f9a6613bfa1af1d08516071a59ae4b0bd09835f56f672d9fe49490", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.3.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "75edd8cadf04e5bf1300c5d62b1fc8118a287bd5d2522db2f2266c17defd481d", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "e4123fdce0f9a6613bfa1af1d08516071a59ae4b0bd09835f56f672d9fe49490", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "44d8e31d6bef0656d7f18600d594e7e0330b4cb33630aae2e0674e58eeb472b8", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 81, + 57, + 131, + 233, + 68, + 129, + 94, + 145, + 208, + 262, + 184, + 48, + 211, + 51, + 44, + 72 + ], + "source_correction": { + "name": "model.language_model.layers.7.mlp.gate.e_score_correction_bias", + "shard": "model-00059-of-00062.safetensors" + }, + "source_layer": 7, + "source_weight": { + "name": "model.language_model.layers.7.mlp.gate.weight", + "shard": "model-00059-of-00062.safetensors" + }, + "target_layer": 3 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.4.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "473dd0c2ba0cca8380cc1dfb9f64044810b470d4a205b65cb7cc8145d2e1e926", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.4.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "99fd67c97447e3c8e90afe0393962f28cbc16ebcaec30985c75633f8a1e98e03", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "473dd0c2ba0cca8380cc1dfb9f64044810b470d4a205b65cb7cc8145d2e1e926", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "0e30b7d85ccd484271da95a8b258762e463782913cdef35bbc2f5cce290f6847", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 129, + 142, + 181, + 82, + 74, + 95, + 177, + 88, + 202, + 150, + 6, + 239, + 98, + 271, + 221, + 219 + ], + "source_correction": { + "name": "model.language_model.layers.9.mlp.gate.e_score_correction_bias", + "shard": "model-00061-of-00062.safetensors" + }, + "source_layer": 9, + "source_weight": { + "name": "model.language_model.layers.9.mlp.gate.weight", + "shard": "model-00061-of-00062.safetensors" + }, + "target_layer": 4 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.5.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "11dbdd083ae664dc6085bf6a64ef6c7881c41ae616bdb4e94115574aff53b7b2", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.5.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "f524352b172de9feadd6d6095b7433ea032ae7b37d8939e49f6b4bb9b050dad9", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "11dbdd083ae664dc6085bf6a64ef6c7881c41ae616bdb4e94115574aff53b7b2", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "428711bd33ff75099827fb0bdec9ac7dde75cf60a6cae71d1cfd46488d438da6", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 96, + 98, + 285, + 75, + 81, + 284, + 111, + 9, + 177, + 268, + 0, + 159, + 70, + 28, + 65, + 144 + ], + "source_correction": { + "name": "model.language_model.layers.10.mlp.gate.e_score_correction_bias", + "shard": "model-00004-of-00062.safetensors" + }, + "source_layer": 10, + "source_weight": { + "name": "model.language_model.layers.10.mlp.gate.weight", + "shard": "model-00004-of-00062.safetensors" + }, + "target_layer": 5 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.6.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "6f7f0f9a3092b8c194c6e450baab07d67593c2e645e1ac3a2990d04bc9d7f7b3", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.6.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "2d9f1c7ef8e374e2804da775e40f9ea12cf13b211bc0cd1add1af09e5a900cda", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "6f7f0f9a3092b8c194c6e450baab07d67593c2e645e1ac3a2990d04bc9d7f7b3", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "46730f97caf95d1bd5213e4cc42c4872e41c6f513d0a8ebbbd2d2ae5560b41a0", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 246, + 103, + 109, + 138, + 128, + 69, + 178, + 219, + 28, + 159, + 220, + 66, + 81, + 91, + 166, + 47 + ], + "source_correction": { + "name": "model.language_model.layers.11.mlp.gate.e_score_correction_bias", + "shard": "model-00005-of-00062.safetensors" + }, + "source_layer": 11, + "source_weight": { + "name": "model.language_model.layers.11.mlp.gate.weight", + "shard": "model-00005-of-00062.safetensors" + }, + "target_layer": 6 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.7.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "f4d8ab11092e879abab8905a345528698b32e3186639ab3ffd85f0afae53fcbe", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.7.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "053fc9ec064bb36ab561bf24fc21654b81dd5f61018407e50319ec1f2f2b4ee3", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "f4d8ab11092e879abab8905a345528698b32e3186639ab3ffd85f0afae53fcbe", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "ab3f340e5e5499f01e402605d863bdd69063414387835f8f7c4f6b1a9f18b8b9", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 178, + 145, + 13, + 25, + 75, + 187, + 70, + 169, + 235, + 20, + 142, + 209, + 156, + 52, + 99, + 177 + ], + "source_correction": { + "name": "model.language_model.layers.13.mlp.gate.e_score_correction_bias", + "shard": "model-00008-of-00062.safetensors" + }, + "source_layer": 13, + "source_weight": { + "name": "model.language_model.layers.13.mlp.gate.weight", + "shard": "model-00008-of-00062.safetensors" + }, + "target_layer": 7 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.8.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "d09b434718801ddeee1ef2d1f1a7c8994ced27ea2acd78663bb8510b30428dca", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.8.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "ce989e6733886f46e30c1561af492976ebc7edeb8f228df9999b375c80b21c30", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "d09b434718801ddeee1ef2d1f1a7c8994ced27ea2acd78663bb8510b30428dca", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "d2286f730bd5736588dd7faf5f6204db40acd90be6afd57a653a080b5c030c29", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 143, + 121, + 277, + 228, + 46, + 72, + 285, + 243, + 70, + 180, + 86, + 23, + 150, + 205, + 28, + 275 + ], + "source_correction": { + "name": "model.language_model.layers.14.mlp.gate.e_score_correction_bias", + "shard": "model-00009-of-00062.safetensors" + }, + "source_layer": 14, + "source_weight": { + "name": "model.language_model.layers.14.mlp.gate.weight", + "shard": "model-00009-of-00062.safetensors" + }, + "target_layer": 8 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.9.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "a7d957beb41c53c65f4643858f586f4df6273e360228452d9b1b6675f47e45f5", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.9.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "a99c13419dce684a0f35aaf8a0c752a3b6e19f821f8c81533a4411985e823a14", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "a7d957beb41c53c65f4643858f586f4df6273e360228452d9b1b6675f47e45f5", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "7d2b8384401d98b798e972d31e4ada972d96791a125812682ef3e9ec0e19c7e3", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 229, + 219, + 23, + 224, + 64, + 244, + 273, + 90, + 66, + 185, + 77, + 237, + 96, + 136, + 226, + 109 + ], + "source_correction": { + "name": "model.language_model.layers.16.mlp.gate.e_score_correction_bias", + "shard": "model-00012-of-00062.safetensors" + }, + "source_layer": 16, + "source_weight": { + "name": "model.language_model.layers.16.mlp.gate.weight", + "shard": "model-00012-of-00062.safetensors" + }, + "target_layer": 9 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.10.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "6646d36d11c26aa7a480f8695c54f22b8a1cbb56003be08a84e65ff39506727c", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.10.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "bf4b0733cd31d612992e2588fa4ae77a56e6661cc3a34f7d8035c8cb0b935433", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "6646d36d11c26aa7a480f8695c54f22b8a1cbb56003be08a84e65ff39506727c", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "69434b74f0ca9953ad3c672afb0abc7b1122906c1cd2bee87bb56d684d439dc0", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 279, + 172, + 114, + 217, + 9, + 66, + 157, + 180, + 210, + 102, + 150, + 283, + 175, + 89, + 32, + 186 + ], + "source_correction": { + "name": "model.language_model.layers.17.mlp.gate.e_score_correction_bias", + "shard": "model-00014-of-00062.safetensors" + }, + "source_layer": 17, + "source_weight": { + "name": "model.language_model.layers.17.mlp.gate.weight", + "shard": "model-00014-of-00062.safetensors" + }, + "target_layer": 10 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.11.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "a9409b29977b9e7c96d2064f7ba26fefa56fa92aa9aabf47f83f902c3d595b2d", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.11.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "1abca858d757e390e2e15d08defcc62116ecc0011aae1deb318ea7811b9886e2", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "a9409b29977b9e7c96d2064f7ba26fefa56fa92aa9aabf47f83f902c3d595b2d", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "030b3b786b1382c0d7355c7f81762b16e05e2afff5f40b4dcf52ec763bbaef47", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 157, + 281, + 111, + 228, + 27, + 5, + 23, + 21, + 282, + 261, + 92, + 81, + 271, + 85, + 80, + 136 + ], + "source_correction": { + "name": "model.language_model.layers.19.mlp.gate.e_score_correction_bias", + "shard": "model-00017-of-00062.safetensors" + }, + "source_layer": 19, + "source_weight": { + "name": "model.language_model.layers.19.mlp.gate.weight", + "shard": "model-00017-of-00062.safetensors" + }, + "target_layer": 11 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.12.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "1b96071c054128baa00f98d89f475da1c965b98840033bcd8d14f8808205ca6b", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.12.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "3e3d5f0e1e21456c5ee2241762ca6b065f64cf20da3ea33120106f5c57fd2561", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "1b96071c054128baa00f98d89f475da1c965b98840033bcd8d14f8808205ca6b", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "e34ab37e681abc054ebc5da08ad6b537abcf7a2aaa09ba003ac156d2d858d9f9", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 269, + 92, + 42, + 15, + 173, + 216, + 141, + 282, + 29, + 210, + 35, + 110, + 260, + 64, + 245, + 247 + ], + "source_correction": { + "name": "model.language_model.layers.20.mlp.gate.e_score_correction_bias", + "shard": "model-00018-of-00062.safetensors" + }, + "source_layer": 20, + "source_weight": { + "name": "model.language_model.layers.20.mlp.gate.weight", + "shard": "model-00018-of-00062.safetensors" + }, + "target_layer": 12 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.13.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "6c7c22f8efb6932231ae0bd3413e20fd177bc05165ce855850635b5edcead72b", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.13.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "c66732b94285cce16ef7f9395840502c036fa0dfdca668b0426e31b1cd3790a8", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "6c7c22f8efb6932231ae0bd3413e20fd177bc05165ce855850635b5edcead72b", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "10b6a7e49e315a5f1c504bb2f1dd06491688370153ccb596b27b72efcb9242a4", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 123, + 165, + 166, + 178, + 5, + 182, + 33, + 37, + 110, + 211, + 186, + 158, + 63, + 106, + 159, + 267 + ], + "source_correction": { + "name": "model.language_model.layers.21.mlp.gate.e_score_correction_bias", + "shard": "model-00019-of-00062.safetensors" + }, + "source_layer": 21, + "source_weight": { + "name": "model.language_model.layers.21.mlp.gate.weight", + "shard": "model-00019-of-00062.safetensors" + }, + "target_layer": 13 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.14.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "cce595ac0f2cff6685392619bb2cccbcb64d846482e67dd63771d21826d981ac", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.14.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "3d135756913b703c8ae6f27242958ec1d77537f0d605957f6af13377a570b4d6", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "cce595ac0f2cff6685392619bb2cccbcb64d846482e67dd63771d21826d981ac", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "6ab2557ed4ce7140a0f34fb4680f191ab05216be6081247b64265a6e5728ff36", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 269, + 234, + 202, + 229, + 106, + 90, + 67, + 11, + 133, + 152, + 254, + 70, + 219, + 13, + 57, + 10 + ], + "source_correction": { + "name": "model.language_model.layers.23.mlp.gate.e_score_correction_bias", + "shard": "model-00022-of-00062.safetensors" + }, + "source_layer": 23, + "source_weight": { + "name": "model.language_model.layers.23.mlp.gate.weight", + "shard": "model-00022-of-00062.safetensors" + }, + "target_layer": 14 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.15.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "9adad27c0753af29820c97589eb3007662a64df075cd6882d325760b29853e66", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.15.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "bf2d12b1f5e7aefbaa945483936f6bd495277ba1bfa2e9329211fcc9a8da5bea", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "9adad27c0753af29820c97589eb3007662a64df075cd6882d325760b29853e66", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "6da7556b69900077ac6840d3d88e121d999c0325f1fb31741a457a53c590a4df", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 135, + 49, + 193, + 207, + 272, + 25, + 45, + 101, + 177, + 56, + 29, + 23, + 119, + 198, + 35, + 159 + ], + "source_correction": { + "name": "model.language_model.layers.24.mlp.gate.e_score_correction_bias", + "shard": "model-00024-of-00062.safetensors" + }, + "source_layer": 24, + "source_weight": { + "name": "model.language_model.layers.24.mlp.gate.weight", + "shard": "model-00024-of-00062.safetensors" + }, + "target_layer": 15 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.16.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "81183eec92cb10fb4892ad6440033930be21688f64049d65b99982f326cf7687", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.16.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "0f534f62cafeb1029a8a7c48cfcb21e4e22018f85a7fe26671f560e99875f982", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "81183eec92cb10fb4892ad6440033930be21688f64049d65b99982f326cf7687", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "0a93b0abe393ac0198d0d6e3b936c686bd7134a595d976ddf4680f5b75b68a6a", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 227, + 49, + 43, + 214, + 245, + 127, + 248, + 150, + 25, + 223, + 246, + 203, + 145, + 57, + 89, + 208 + ], + "source_correction": { + "name": "model.language_model.layers.26.mlp.gate.e_score_correction_bias", + "shard": "model-00026-of-00062.safetensors" + }, + "source_layer": 26, + "source_weight": { + "name": "model.language_model.layers.26.mlp.gate.weight", + "shard": "model-00026-of-00062.safetensors" + }, + "target_layer": 16 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.17.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "4d7db39d2e2c611c9f01a3918efb990b84db4ff39f228b1bd1bf43994db86bd2", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.17.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "fc4369f5e0a20f4685f80ff81ee5dfc599f218d8d164befe751f85ddb2dc7a2b", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "4d7db39d2e2c611c9f01a3918efb990b84db4ff39f228b1bd1bf43994db86bd2", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "9222cfecbf0d01ee54ac2d86f67bcc1974059ab1ef262ef6b4f07832a9ff370d", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 134, + 196, + 89, + 66, + 106, + 205, + 60, + 121, + 209, + 224, + 286, + 114, + 26, + 248, + 188, + 194 + ], + "source_correction": { + "name": "model.language_model.layers.27.mlp.gate.e_score_correction_bias", + "shard": "model-00028-of-00062.safetensors" + }, + "source_layer": 27, + "source_weight": { + "name": "model.language_model.layers.27.mlp.gate.weight", + "shard": "model-00028-of-00062.safetensors" + }, + "target_layer": 17 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.18.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "7c7a03b79a47398e55a4903dd0612659338d63b00a4141448d93ca43d306c9fc", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.18.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "05dd881b3b10c8b08d508b1aee7d2f29d57935dbe39b33cb38f382443a49dc04", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "7c7a03b79a47398e55a4903dd0612659338d63b00a4141448d93ca43d306c9fc", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "6543ecf00c8031aeb77bc1f7d05148aecb76d560acd8367a6e9ad03af0652f2d", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 69, + 214, + 198, + 283, + 272, + 56, + 89, + 107, + 60, + 50, + 245, + 61, + 185, + 67, + 97, + 99 + ], + "source_correction": { + "name": "model.language_model.layers.28.mlp.gate.e_score_correction_bias", + "shard": "model-00029-of-00062.safetensors" + }, + "source_layer": 28, + "source_weight": { + "name": "model.language_model.layers.28.mlp.gate.weight", + "shard": "model-00029-of-00062.safetensors" + }, + "target_layer": 18 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.19.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "8caf87a0be91376736e8c2a020d56ccd324dffaaf3b6e7ee89c8b1d587ef9f00", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.19.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "08f49ac445fa5d4e525f35a853be6860520a43069bed14d88296651dbd290cc8", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "8caf87a0be91376736e8c2a020d56ccd324dffaaf3b6e7ee89c8b1d587ef9f00", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "4ace814973b9f159458a5f56db27ba7860020cf772bf689bf602f5de60ceabd7", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 93, + 139, + 36, + 279, + 29, + 97, + 111, + 64, + 0, + 195, + 242, + 24, + 140, + 101, + 8, + 136 + ], + "source_correction": { + "name": "model.language_model.layers.30.mlp.gate.e_score_correction_bias", + "shard": "model-00033-of-00062.safetensors" + }, + "source_layer": 30, + "source_weight": { + "name": "model.language_model.layers.30.mlp.gate.weight", + "shard": "model-00033-of-00062.safetensors" + }, + "target_layer": 19 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.20.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "8f3871569ca10fa5f9189f855525b36c9386592cd22de87d4a8a694c73125182", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.20.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "6a7668da41f9cf8bd9a48e1ab23540d27028f95e2b019026633795ff4f42b30a", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "8f3871569ca10fa5f9189f855525b36c9386592cd22de87d4a8a694c73125182", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "33780982419afc3d21872e1ee68211c4126a737463a4fe54e32f9195facdbe6d", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 147, + 241, + 11, + 251, + 178, + 268, + 118, + 154, + 103, + 248, + 25, + 157, + 197, + 66, + 210, + 98 + ], + "source_correction": { + "name": "model.language_model.layers.31.mlp.gate.e_score_correction_bias", + "shard": "model-00035-of-00062.safetensors" + }, + "source_layer": 31, + "source_weight": { + "name": "model.language_model.layers.31.mlp.gate.weight", + "shard": "model-00035-of-00062.safetensors" + }, + "target_layer": 20 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.21.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "363f5bf298d05089055267b9e963eb0ac9cecefcd00fce1c79893810a649627a", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.21.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "3fa5d3a400e2dd8495d494f5cd2781b02bc5c2005219201813f30b3610d88f29", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "363f5bf298d05089055267b9e963eb0ac9cecefcd00fce1c79893810a649627a", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "5ff9a7b4218b26f2c1ad30de4eb59570f435f737837711a687ff7d35eac15c83", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 206, + 72, + 170, + 171, + 237, + 8, + 168, + 61, + 172, + 202, + 130, + 222, + 256, + 128, + 226, + 252 + ], + "source_correction": { + "name": "model.language_model.layers.33.mlp.gate.e_score_correction_bias", + "shard": "model-00038-of-00062.safetensors" + }, + "source_layer": 33, + "source_weight": { + "name": "model.language_model.layers.33.mlp.gate.weight", + "shard": "model-00038-of-00062.safetensors" + }, + "target_layer": 21 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.22.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "d3492404e2cb487278366cafc2930407e9756b0a9c7c59ae14cf195a1828a07c", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.22.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "fa3c81ced819540bc8cb3a85526f0e936c0f8042b66d517ec43d456cd7581aec", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "d3492404e2cb487278366cafc2930407e9756b0a9c7c59ae14cf195a1828a07c", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "55bed9fff3b26ebac695ece4d0ee6e42fb0eaa5a9187f1ebce26d49c30875426", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 52, + 258, + 18, + 59, + 80, + 172, + 137, + 40, + 197, + 89, + 35, + 154, + 84, + 19, + 255, + 38 + ], + "source_correction": { + "name": "model.language_model.layers.34.mlp.gate.e_score_correction_bias", + "shard": "model-00039-of-00062.safetensors" + }, + "source_layer": 34, + "source_weight": { + "name": "model.language_model.layers.34.mlp.gate.weight", + "shard": "model-00039-of-00062.safetensors" + }, + "target_layer": 22 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.23.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "653843a86ffcfc81999e775825df3819a8beb7fe422890cf2816a57c86cb6574", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.23.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "e9c0aa84708bb619657fc5c19da2c936de328ec89a58e21bf3e11db73d5d8309", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "653843a86ffcfc81999e775825df3819a8beb7fe422890cf2816a57c86cb6574", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "18486f9a8899fb669456ca15c6c1633d77852ccae9b6b13ef954205f2baa87bc", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 220, + 31, + 280, + 106, + 10, + 178, + 234, + 36, + 7, + 212, + 187, + 199, + 221, + 47, + 186, + 147 + ], + "source_correction": { + "name": "model.language_model.layers.36.mlp.gate.e_score_correction_bias", + "shard": "model-00042-of-00062.safetensors" + }, + "source_layer": 36, + "source_weight": { + "name": "model.language_model.layers.36.mlp.gate.weight", + "shard": "model-00042-of-00062.safetensors" + }, + "target_layer": 23 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.24.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "9e42b64e06df0c4c237412648b93ad6bba047b8a606c5cb8e7e3d6343f26d850", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.24.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "2d509731469d100f7f8b5a7a5f5b87f0f9461bccc37c5b01473eae549501cf11", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "9e42b64e06df0c4c237412648b93ad6bba047b8a606c5cb8e7e3d6343f26d850", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "06d8db479b79cac807c170da0be244f36fed2d7878c0bfa106c488f7833ff224", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 36, + 197, + 33, + 93, + 181, + 134, + 84, + 244, + 264, + 164, + 207, + 202, + 71, + 59, + 214, + 284 + ], + "source_correction": { + "name": "model.language_model.layers.37.mlp.gate.e_score_correction_bias", + "shard": "model-00043-of-00062.safetensors" + }, + "source_layer": 37, + "source_weight": { + "name": "model.language_model.layers.37.mlp.gate.weight", + "shard": "model-00043-of-00062.safetensors" + }, + "target_layer": 24 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.25.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "5fd8aec57c118230902d2aa431056e1d1229bae8c066416b102a9bfaf3fcbc67", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.25.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "cc56e6d1e1a3aa73ff5773afdb13e0215580ee922b78b72f7d1fbc433ce1f1c4", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "5fd8aec57c118230902d2aa431056e1d1229bae8c066416b102a9bfaf3fcbc67", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "1262ffd24ea87882179c7d3ec01e99219b35b931b00de18c2184e7787eab3e1c", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 54, + 143, + 130, + 15, + 14, + 107, + 76, + 225, + 8, + 195, + 281, + 212, + 161, + 177, + 258, + 259 + ], + "source_correction": { + "name": "model.language_model.layers.38.mlp.gate.e_score_correction_bias", + "shard": "model-00045-of-00062.safetensors" + }, + "source_layer": 38, + "source_weight": { + "name": "model.language_model.layers.38.mlp.gate.weight", + "shard": "model-00045-of-00062.safetensors" + }, + "target_layer": 25 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.26.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "e0b58b08a4ffff09b3101994cf1256d00354d8ac0bb46b55936be1f9f396983e", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.26.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "39f637376a2fafd924ef8feb5a9df4895a87c90e99622460d59e31f1e641ed83", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "e0b58b08a4ffff09b3101994cf1256d00354d8ac0bb46b55936be1f9f396983e", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "89fd37f3e59bff66abf3d8d1ffafd8a808d60c062cdf68f351a6d1d77d4609ca", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 5, + 187, + 78, + 105, + 165, + 9, + 156, + 139, + 235, + 121, + 62, + 256, + 14, + 201, + 180, + 277 + ], + "source_correction": { + "name": "model.language_model.layers.40.mlp.gate.e_score_correction_bias", + "shard": "model-00049-of-00062.safetensors" + }, + "source_layer": 40, + "source_weight": { + "name": "model.language_model.layers.40.mlp.gate.weight", + "shard": "model-00049-of-00062.safetensors" + }, + "target_layer": 26 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.27.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "bb733d9afaff19d3aa688e8d3dc6a66ad0be6ff12412dd0500e49ac5ba2b6772", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.27.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "a1faacf3027b487f1b2e0a82decdc3b52bf00db21fabeedfba7ca1d369e3ee41", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "bb733d9afaff19d3aa688e8d3dc6a66ad0be6ff12412dd0500e49ac5ba2b6772", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "695fcf8639f40d3522b8309871c6dd3e84af45ba64df1fc808fdf6ce0d2ff5b1", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 144, + 154, + 125, + 98, + 92, + 123, + 196, + 80, + 141, + 83, + 249, + 250, + 20, + 34, + 134, + 184 + ], + "source_correction": { + "name": "model.language_model.layers.41.mlp.gate.e_score_correction_bias", + "shard": "model-00050-of-00062.safetensors" + }, + "source_layer": 41, + "source_weight": { + "name": "model.language_model.layers.41.mlp.gate.weight", + "shard": "model-00050-of-00062.safetensors" + }, + "target_layer": 27 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.28.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "e8e4fbfad33424193c1adc1f54365a78fff8e8b1ea6c811686329bd3617d4d1e", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.28.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "db6eaf64743eebe50b26412c9c4e3b7657f1338a77c4c37540276f3bb8ef9ac1", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "e8e4fbfad33424193c1adc1f54365a78fff8e8b1ea6c811686329bd3617d4d1e", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "a0f25e45e50b40e8eb0d3c0d838633ecb829fec889e9afefa32fb6c49a4e6480", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 249, + 148, + 151, + 221, + 18, + 87, + 239, + 105, + 207, + 35, + 147, + 20, + 84, + 130, + 52, + 17 + ], + "source_correction": { + "name": "model.language_model.layers.43.mlp.gate.e_score_correction_bias", + "shard": "model-00053-of-00062.safetensors" + }, + "source_layer": 43, + "source_weight": { + "name": "model.language_model.layers.43.mlp.gate.weight", + "shard": "model-00053-of-00062.safetensors" + }, + "target_layer": 28 + }, + { + "output_correction": { + "dtype": "torch.float32", + "key": "model.layers.29.feed_forward.router.e_score_correction_bias", + "numel": 16, + "sha256": "1d68e79658f3ebf8642dabf9c5b36323049fe3a147a08898cf53c86080c4509f", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "output_weight": { + "dtype": "torch.float32", + "key": "model.layers.29.feed_forward.router.proj.weight", + "numel": 32768, + "sha256": "19c8e55d471d32d06afd61324724cb5cd0f419a4f27e535f26d44ceec62d34f0", + "shape": [ + 16, + 2048 + ], + "storage_bytes": 131072 + }, + "selected_correction": { + "dtype": "torch.float32", + "numel": 16, + "sha256": "1d68e79658f3ebf8642dabf9c5b36323049fe3a147a08898cf53c86080c4509f", + "shape": [ + 16 + ], + "storage_bytes": 64 + }, + "selected_router_rows": { + "dtype": "torch.bfloat16", + "numel": 65536, + "sha256": "d05d18d46082de714b56c1204a1dafc08f54761bcaed6a7d73a14bc35a888ba5", + "shape": [ + 16, + 4096 + ], + "storage_bytes": 131072 + }, + "slot_source_expert_ids": [ + 44, + 212, + 152, + 127, + 244, + 138, + 156, + 266, + 24, + 135, + 113, + 185, + 147, + 186, + 17, + 184 + ], + "source_correction": { + "name": "model.language_model.layers.44.mlp.gate.e_score_correction_bias", + "shard": "model-00054-of-00062.safetensors" + }, + "source_layer": 44, + "source_weight": { + "name": "model.language_model.layers.44.mlp.gate.weight", + "shard": "model-00054-of-00062.safetensors" + }, + "target_layer": 29 + } + ], + "verification": [ + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 2.7922825813293457, + "relative_to_reference_max": 2.561544959980983e-07, + "seed": 101234, + "source_layer": 3, + "target_layer": 0, + "threshold": 0.002558456516265869, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 4.603104591369629, + "relative_to_reference_max": 1.5538550626151795e-07, + "seed": 101235, + "source_layer": 4, + "target_layer": 1, + "threshold": 0.0029206209182739257, + "tokens": 2 + }, + { + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 5.030343055725098, + "relative_to_reference_max": 1.6588630588638552e-07, + "seed": 101236, + "source_layer": 6, + "target_layer": 2, + "threshold": 0.0030060686111450198, + "tokens": 2 + }, + { + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 4.8936004638671875, + "relative_to_reference_max": 1.9488193272987493e-07, + "seed": 101237, + "source_layer": 7, + "target_layer": 3, + "threshold": 0.0029787200927734376, + "tokens": 2 + }, + { + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 6.121592998504639, + "relative_to_reference_max": 1.5578858585325914e-07, + "seed": 101238, + "source_layer": 9, + "target_layer": 4, + "threshold": 0.003224318599700928, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 3.983003616333008, + "relative_to_reference_max": 1.7957697411362003e-07, + "seed": 101239, + "source_layer": 10, + "target_layer": 5, + "threshold": 0.0027966007232666018, + "tokens": 2 + }, + { + "max_absolute_error": 1.0132789611816406e-06, + "passed": true, + "reference_max_absolute": 4.523782730102539, + "relative_to_reference_max": 2.2398930754100848e-07, + "seed": 101240, + "source_layer": 11, + "target_layer": 6, + "threshold": 0.002904756546020508, + "tokens": 2 + }, + { + "max_absolute_error": 5.364418029785156e-07, + "passed": true, + "reference_max_absolute": 2.9276320934295654, + "relative_to_reference_max": 1.8323402185077925e-07, + "seed": 101241, + "source_layer": 13, + "target_layer": 7, + "threshold": 0.002585526418685913, + "tokens": 2 + }, + { + "max_absolute_error": 5.960464477539062e-07, + "passed": true, + "reference_max_absolute": 3.1342387199401855, + "relative_to_reference_max": 1.901726387214313e-07, + "seed": 101242, + "source_layer": 14, + "target_layer": 8, + "threshold": 0.0026268477439880373, + "tokens": 2 + }, + { + "max_absolute_error": 4.76837158203125e-07, + "passed": true, + "reference_max_absolute": 2.2202911376953125, + "relative_to_reference_max": 2.1476334797160313e-07, + "seed": 101243, + "source_layer": 16, + "target_layer": 9, + "threshold": 0.0024440582275390625, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 3.5541903972625732, + "relative_to_reference_max": 2.012429435000374e-07, + "seed": 101244, + "source_layer": 17, + "target_layer": 10, + "threshold": 0.0027108380794525146, + "tokens": 2 + }, + { + "max_absolute_error": 5.960464477539062e-07, + "passed": true, + "reference_max_absolute": 2.611114978790283, + "relative_to_reference_max": 2.282727695239417e-07, + "seed": 101245, + "source_layer": 19, + "target_layer": 11, + "threshold": 0.002522222995758057, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 3.666553497314453, + "relative_to_reference_max": 1.9507576742806906e-07, + "seed": 101246, + "source_layer": 20, + "target_layer": 12, + "threshold": 0.0027333106994628905, + "tokens": 2 + }, + { + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 3.575976848602295, + "relative_to_reference_max": 2.333530283289243e-07, + "seed": 101247, + "source_layer": 21, + "target_layer": 13, + "threshold": 0.0027151953697204592, + "tokens": 2 + }, + { + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 2.4934637546539307, + "relative_to_reference_max": 3.346609812546823e-07, + "seed": 101248, + "source_layer": 23, + "target_layer": 14, + "threshold": 0.002498692750930786, + "tokens": 2 + }, + { + "max_absolute_error": 9.5367431640625e-07, + "passed": true, + "reference_max_absolute": 2.8226850032806396, + "relative_to_reference_max": 3.3786069479869374e-07, + "seed": 101249, + "source_layer": 24, + "target_layer": 15, + "threshold": 0.002564537000656128, + "tokens": 2 + }, + { + "max_absolute_error": 8.344650268554688e-07, + "passed": true, + "reference_max_absolute": 3.6396937370300293, + "relative_to_reference_max": 2.2926792393702548e-07, + "seed": 101250, + "source_layer": 26, + "target_layer": 16, + "threshold": 0.002727938747406006, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 2.5931339263916016, + "relative_to_reference_max": 2.7582676313983536e-07, + "seed": 101251, + "source_layer": 27, + "target_layer": 17, + "threshold": 0.00251862678527832, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 2.8808417320251465, + "relative_to_reference_max": 2.4828012221340737e-07, + "seed": 101252, + "source_layer": 28, + "target_layer": 18, + "threshold": 0.0025761683464050294, + "tokens": 2 + }, + { + "max_absolute_error": 5.066394805908203e-07, + "passed": true, + "reference_max_absolute": 3.7134902477264404, + "relative_to_reference_max": 1.364321559484388e-07, + "seed": 101253, + "source_layer": 30, + "target_layer": 19, + "threshold": 0.002742698049545288, + "tokens": 2 + }, + { + "max_absolute_error": 5.960464477539062e-07, + "passed": true, + "reference_max_absolute": 2.5689773559570312, + "relative_to_reference_max": 2.320170111160278e-07, + "seed": 101254, + "source_layer": 31, + "target_layer": 20, + "threshold": 0.0025137954711914063, + "tokens": 2 + }, + { + "max_absolute_error": 7.748603820800781e-07, + "passed": true, + "reference_max_absolute": 4.2798686027526855, + "relative_to_reference_max": 1.8104770356307452e-07, + "seed": 101255, + "source_layer": 33, + "target_layer": 21, + "threshold": 0.002855973720550537, + "tokens": 2 + }, + { + "max_absolute_error": 6.556510925292969e-07, + "passed": true, + "reference_max_absolute": 2.8459293842315674, + "relative_to_reference_max": 2.3038206645676488e-07, + "seed": 101256, + "source_layer": 34, + "target_layer": 22, + "threshold": 0.0025691858768463133, + "tokens": 2 + }, + { + "max_absolute_error": 5.364418029785156e-07, + "passed": true, + "reference_max_absolute": 2.640001058578491, + "relative_to_reference_max": 2.0319757116588534e-07, + "seed": 101257, + "source_layer": 36, + "target_layer": 23, + "threshold": 0.0025280002117156982, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 4.370778560638428, + "relative_to_reference_max": 1.63644926729075e-07, + "seed": 101258, + "source_layer": 37, + "target_layer": 24, + "threshold": 0.0028741557121276856, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 2.1171510219573975, + "relative_to_reference_max": 3.378387889605545e-07, + "seed": 101259, + "source_layer": 38, + "target_layer": 25, + "threshold": 0.0024234302043914797, + "tokens": 2 + }, + { + "max_absolute_error": 4.76837158203125e-07, + "passed": true, + "reference_max_absolute": 3.2437965869903564, + "relative_to_reference_max": 1.4699971019007135e-07, + "seed": 101260, + "source_layer": 40, + "target_layer": 26, + "threshold": 0.0026487593173980715, + "tokens": 2 + }, + { + "max_absolute_error": 7.152557373046875e-07, + "passed": true, + "reference_max_absolute": 2.252269983291626, + "relative_to_reference_max": 3.175710472593354e-07, + "seed": 101261, + "source_layer": 41, + "target_layer": 27, + "threshold": 0.0024504539966583253, + "tokens": 2 + }, + { + "max_absolute_error": 4.76837158203125e-07, + "passed": true, + "reference_max_absolute": 2.5564804077148438, + "relative_to_reference_max": 1.8652095152544295e-07, + "seed": 101262, + "source_layer": 43, + "target_layer": 28, + "threshold": 0.0025112960815429687, + "tokens": 2 + }, + { + "max_absolute_error": 4.76837158203125e-07, + "passed": true, + "reference_max_absolute": 3.3981733322143555, + "relative_to_reference_max": 1.4032161152074108e-07, + "seed": 101263, + "source_layer": 44, + "target_layer": 29, + "threshold": 0.002679634666442871, + "tokens": 2 + } + ], + "weight_output_dtype": "float32" + }, + "schema": "glm-folded-expert-provenance", + "schema_version": 1, + "source": { + "index_sha256": "3c3f40366a53c3fd7974b4eab7881a365a98c2a4329150befebab99fe7c18b05", + "manifest_path": "\\manifests\\glm_selected_experts_fixed16_top4.json", + "manifest_sha256": "ef33fc60756901a8a183326f29b96589cb40b69bccbfd02dfad1dca8afe0f391", + "model_dir": "\\models\\zai-org\\GLM-5.3-Flash", + "repo_id": "zai-org/GLM-5.3-Flash", + "revision": "3f1971b7b5f7a528c9c4ef6212c8785298a8c24a", + "selected_tensor_fingerprints": { + "model.language_model.layers.10.mlp.experts.0.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4c79341a8bd551af40bb106bc743a3de7be0cc988822b5327e038472c12d6f05", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.0.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb35fd659d305cd64667f5e79711393b917c74d748d9c32736fa027bbf5896a4", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.0.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "998b9bd4ece47a36db05610275beaa6ea509d671ae30ed7763ab151c9f6a235c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.0.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ef06fa7b434bc9026f28fbc518d94b94a7c76850b90397dbf0c017a1650d720", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.0.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "910d2867d4e233bb084fc074d64d6d492338c260b2fe261ded0405b3d61ac9a6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.0.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "74774aefc78f494c98e6b339c06e9e3d3f4fe1d667860fdd16e6111cb8ec8247", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.111.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c28c06114c64880016bf2847d8efaf899c0dd5656f55db6182a856142142fcd1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.111.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9293283004eccac66739b64d1ecf3bdfdc9e29babcbbf7844ab034b16944e224", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.111.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "faa819805529de53b4f9ec104b4ba11104081a882fc0cdecf57d89cd4bf9c85e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.111.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "22b9f4becc54011dc250fdebb50d1a46228c54be8e09f3f3440c6a63192caf0e", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.111.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "acda93b779d4b497e95d6cf30ed07a9f99cdacf9f13b74171284d0ef16abd188", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.111.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b61e82e06c14300856591f96d55622feb4e789fb165f66730a8af7514f62bba", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.144.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bdeb6b88f21e0b27fc36a4b067faa28393c4d5a811c2255ef9db29638b5c6178", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.144.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a7a00bd46bfbe39c849994d4738d4b9f327747c21f8af207d9f11eb491a0b0ca", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.144.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "270628f5de9eca0208b0de9402dd794f6d3f321bf8bbced85c364bc30bb48171", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.144.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e0886e5147649bc65de36703b27570267e178982c5acddf9ef8e1a8619f25cb4", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.144.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9dabe33fc39bdc39a7fae438d375ad1def334d97731294d912aeb380cfd05ec5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.144.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eacd52553dde9e6ce19f4cc5a9f5e7498514ec6ea77b8afdc6b0af2320da60a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.159.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "15687ed9716feb2f02a0c39473547cda9fd369ec72c88b12080aca57fe84eee7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.159.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "db2880d80918390458284c687494172e570c364206d77d394f2b4ed595985d28", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.159.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "adc66fe73b671daf66ce5ef2ed35eb79b0905f0f00c7a497ec874e9f3e600961", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.159.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1f5aa1d371ccaf775de910b6141b4ef0b83f3d3abd78b05702be1e63e636cabb", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.159.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a2ea9ab8d4a615222e68290b050c28a46a27c5368d8f0fb2d17e4e532962ef77", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.159.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "83b6520e895c6aad6075edbd89be1a5b2132d52a9f7e3e6096e330ea55581d78", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "34363757ffb34ac6b55af49f571ca23479cb2918a09f1d9d19b98b87f2ec53c1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16e0a42847815dcee5e09c1ea7372b3e40669273ab1ef908e9afa97387b87bc7", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "561523d4cccd04b209115a0b5ce006b4e5d8b412881e564d571eb80312a42957", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38a5d24602bde37e3a81b098fc79c456bf2229f49374e8f6162500d118659d2b", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "50c19167544c4d67b62f90a0f78864c256eb0f30cfe761169065eb739ef8925c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5bd9d5cd6cfdbdb3887a0b8ef6bd3b78326e5dee7ca133ce731cec1cbda85ea4", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.268.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f0f8e88a6f6994606e5c66defb698609d4c3ad97e258d6f727461db64ee5eba9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.268.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a7f11306c9fcb97deaf7b43cd1ddc69fc3119040eb24677b07cbf5580c0a703d", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.268.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8047692b6d3019d484cad509e475dd1b3dd44bbf616bef21f92e48042979d39d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.268.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eba70b30f36e87a472c67f72dd3eb51b69df3406efbb2683d68ac2b78f191038", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.268.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cf86e318af9a89344b03903b456ce266b5484a1c822c4f75f54bd369ea548088", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.268.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "57d304cfa4358561280991663bbe5cdb861999a11cbb4d52bfc96a3206023273", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.28.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "791c173bd24b85684d96458843c278fb1b8054738cb99ea1cfe37e91b68240b4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.28.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f2353d79a6db7e241bffb2c354ac7cbd368de5df5c7452f52b3d617a53150e42", + "shape": [ + 32, + 16 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.28.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cee215d06c2f2326cda8e41050ecb5073e53ef8ded36e8265e43b151620b0692", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.28.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "da4329c91dc9dd518ad0652a2e87dbe0687d3c1855d31333858252a084b76850", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.28.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "316ecdfddd4a4bee3705818a3719bf2aa3270b797e98e87b9e21561094e59482", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.28.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "47566d51763cf3f8848c59de5cc9a698e5808a0630be3b7a2d4bb47794076478", + "shape": [ + 16, + 32 + ], + "shard": "model-00003-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.284.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df2409a215e010e47fb913fd1948f8e513c1aa93110fbe46233c72debc1882e0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.284.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b3deae81fcdd056b8439a8180d20ff3f7142b573d9f990dd79448ddc90caea06", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.284.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4981aea1040d8ff0acebf90e3f07819f6187ef09d418192f2e7e322a40421650", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.284.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ffa869c9a169c98ba1f02e9b95b196dc9821c099c30f4190deed89e3ddea3904", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.284.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0a23030e62e14d63192baee18cf235c405afb5033dd6a866c4247fac22e2e124", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.284.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0b33c556e1097f8357069091403ec679d01ba982ea47c871f4790a450f0c0b45", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.285.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b30ae5457af3f3ab9d1f5e126d9b430bab59e42a659db7c5256382eb512af46e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.285.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fa08761dbd542110e1d29198efdcf404f5d780a95587fa93ec91c9330ac06bf8", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.285.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cae2eed1f64c26f3c43a24ba7b9c5ede5cecbb5a246a0cf6b0555e14c8ab59d0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.285.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4b0b56f0f0533a8604abda011c53a131613df077a29e66701e9a6a690283176", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.285.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4151dc7244ac144d4a6c19f79b75d6dfe5273ea9bccc8e34a08d549db5649547", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.285.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d17dc06ff42cd6ed25afdc17eabda2cd1067a0aeed7b183c7642d692854cda5d", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.65.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "336c45fc872f4fa67959c8a6b36d0aa676dc57278a8471f9cbc9282b270cc9af", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.65.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a27d0ca358b8ec1f1eb45b7d9ae5b5ce65592546b6ff5d4feb45d16e6e9adc01", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.65.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c42f81fadb5bb256cde5b45ff11a51e7d2abcabc81951616512c7e05b87928a9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.65.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d40ce5399abe75171b30b8f72f2d3d1087e47d99b5828908239ad94bb455826d", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.65.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2313acdfdd4a866eaa621ec8c6b8ac9e6bde5725d2f19a32c7b7b3ab7da87c34", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.65.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "061701c27403ef065ef673b07e7a35a3d05f9f3659a82ca16fa69ac42eac5ad8", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.70.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0622279124d55d04c4a29d1a5716b6b90dd92c5809da82dfb3b5ecf720cbf7e2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.70.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "72315278659de7725adce5778fc7d935f7f65983699ce61e5ca79bc700e11e5a", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.70.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "65cab1eab493f2b6585a67b7476d1cdbd3a76d01a5137e1bd5501424deefa910", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.70.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8a6e0b7784ba8d18752dec019507ad75ea718e7e02dadc8f0f8cc6c2c4ff1d18", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.70.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18d52fd352de0ea8dcccb8037c2a79ad08f56c759387ca1dd5612921f8b67d47", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.70.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38333bd8c0830ad0c1c1058d4cc7514cc3bd7dda87a7869fcecb94c9c6514fc0", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.75.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c6091fc20a0e558aefce5403d71bf40f1b4b59ce0961451faf7aa738f1279b9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.75.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "47291d9f1d714343e6aa30f86f80ce6b80edc4421cb7e6ea823e807c604877d6", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.75.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0051632ff43bb59b5e2a7bded39eb8eb3d3d6978a9c0d48f9a569c6ebf27ed3f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.75.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5c2d1b28d6a32e2ba33d7154f7140a753ee18514b021885ebe5712869d5212ae", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.75.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c50b2f4de5d678cc9ec3194f08b4d98954944e01bc25b4604ee37d8d0e555c00", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.75.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e5af4255b38777d60d7a3ba1f517f0fd772874b69e0a819680f45c5bdbfa1356", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.81.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "935a77677eac46fbdbfe456081b783454dbcc0646a4382e50e90aed8a3b216e1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.81.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4c8575d448cf455ef1f96d862316d9a2f8c84bb5c8157b4e622440ab9020afbd", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.81.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d65525664d1caa75a922b443bacacc4f49b1fff63f9f6c409c51cf79bc6fb0ca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.81.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc342eb8123b842697c8155d92fe2fa8322b454f363a57f0c32886f0d7b0ce0a", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.81.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c223e30e2f28fefcf379b435f7a9321ba64eb66ea26311cb4071f1f294bc6cb3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.81.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7cdc38a89f69ef3f038b3424918059b6c173f526b6aa90253f80d8a9f059b972", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.9.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c7b6d87cec06fa31c2ea9fe4d91eb23b36843dd7cc43c41bab40623865df817c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.9.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7480e6cef23da94c7cd66f4e97271eef329f300f9c85c843902ef6e4caea1ab4", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.9.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4ed4e30969a77a5c9c61dfa0e5f72c34e741371a528ebcea33a7c65ef653f5a5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.9.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1536fda42a7ed435281a3152ea8c6d82dfc80deea8fbbb966e364274d66b3eef", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.9.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d3f5aa4bb720a72aa76c53754ab71a3fc88cfd06dc95afc90dd761e4e252ee28", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.9.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f83b692a29199e29beb6b96269be661be9e0e12e531f741838f77c0d873c1577", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.96.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "caa0ab11b1af629731ddbcb80fd2166ce0563eee2e47f833430cb539358d9d3b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.96.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "548bafe3e95c2cdf380d4ec26303a396d8e6beb52726b5055b2868fbd753af81", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.96.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "392ecb0617079636ce1b0043099f48e9ed29c998a89e16fb9183957ee0955d9e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.96.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d8a6f1cae4d1802fcd9b3512cf48f4d5c0c55d4bd3078be2fac551a6d65adc32", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.96.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c03c42dbb82a18d44cff2a54d59d417e9bd55926e09d6a85b7da3ca14afed32", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.96.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5f83f340e49c719552599f752a98e838fb913ea8bcbc777d29831eeee8d82478", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.98.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9d7e761883c042d7c27f5ba12afb48d6d8df7a7a4f82417c6deff3253e627562", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.98.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f9ba2175856eddaae3fb92b1b46745e752704649b4d22a49d00073377a6c5f72", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.98.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a192480261a2a0eccad6ca8bb77f322a365463fe3a54a25ba5aa69bc5737f1ea", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.98.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3f1785783372423923572cff6ddb6b04c38c07106d9859eb0f5ef1c1d6e11d28", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.experts.98.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f7f8d8a04ec1b67abcb692a0703102cfc3a70a4f50e0deae94a28f1f06a90c9b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.10.mlp.experts.98.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2b4720afbfb9ff0703e5e16dbad17cd1c32b6ac82bc9f1df036306bbfa8fc462", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.10.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "bd908d64a7c91a6316715dede1ead4201d8d6bafa0e9c06a02b2ebfe3cb31249", + "shape": [ + 288 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.10.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "00cb736d59b071e484edd72951044d627a25023482093dd0cea795fb94cf19a2", + "shape": [ + 288, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.11.mlp.experts.103.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "58641baeadd04926b686231c6f9416a588a1cd51402c8a4910f75170ecf38040", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.103.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "41070412e3827dea635cbee74956a3a6c3b395bf710c21f8a5b126347d6e9480", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.103.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5cfc1de31f0080a87a2e44f8581904db57a9e6c4cf2035eda79149823bf91503", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.103.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7df120fc441458a2acbc6992839049c00b80536477a4f8ba869f5cb4308f7dcc", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.103.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99b90fdef267c2a9bd3bbb1f977cc3a30f0150e2aebdec7f504863c884ff447a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.103.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a8fe6863dde47bb41cd0d2e7f176b39f8c9d9ea2562bf01d388dcdf0d6d54afb", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.109.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ed2311ea0bec61dbbbf502ea9583ea26a026e37b68446fb7462079b7f36e608", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.109.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0af95a0af9338097e020414fa6bb19a3b40211712396d96914734b129587c66", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.109.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4a3e0de282172d309eb069fa71e5d6e9549b51f0377d63151d8b8d678dbe6ff3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.109.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ae3da70e7ecd65907c50aed2c2a01aae5dc7c60462d41479195e2254beb4849", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.109.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4fdb8709544fd560b96df0023d7cfe6e3826d94d8a06fbe5ffbc40480ad17223", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.109.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "610633b4d846314ebde85d56ed10ab9b82f1cf8718643c0b531f2dd378906e7a", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.128.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1d6362aae2cae480212b5162652423167d314107fbf414d599a89c2e85573120", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.128.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8678e6bf3294c77a926f8161ad687003c32f0d73bdf8e4e112e70ad91f669bef", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.128.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "504949b6f0a1bea512d1fbabafa6de1f46ce77405279cdf53492c154613f08f8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.128.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a229968df11bf6739a05a4180859bf2058141b6f37f4e16781cc57572cafb77", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.128.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17ced02e9fa913b85b5366274f28888680ddf34a733722f7c26b0999b3ab2702", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.128.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fb7242c072f366e822a1553122cfd9ec4831d5830b5da2240e5475a9db7129d5", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.138.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce74e194d4925f06ebdd436df1d90f07a21e459378bbdbcd61eac459e27446d4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.138.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d81d0e991ec7cae74b10035a689978abadf2a82357e3b9513bdf84c67fcb5598", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.138.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e9e1585cb5fbdced1c800ee0d71b47eb84114e8a7e5adcaa3b2f3e5eea3da6e8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.138.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fe393b1de2dc90c0473edafc0c4e7eb8315f0d7098b1f166b2dff8f2156e11d1", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.138.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3bfad1c9ca8155557f8ceee426979437c19495e64b14425199b46459ea4dbbc4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.138.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3d8425aa77777b2a4c0c5955259ee73d60a5ee780d0d3e45449a3965dd6901bd", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.159.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "49ee0e1dba4b83c14df20ecf04f520e5202e8c502a25ba4997e261e1efe98233", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.159.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e6c000943016995adaf76fc5865913ac106c051ebc5b9b20f221444afea82d2f", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.159.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0e8875af727ba2614c4dc494f0f9dd7e3311481674f872e7abc001e3908f218d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.159.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c9bf042a900e7a4a7b074af1da185f5e79365997c9261bd1c95b76c3a53495bd", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.159.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a69b298e007cbd994ebb0ce29f81021876b07b557a57ac9f7737116e65f0e081", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.159.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5b5b0bc16f3ff1e377582fcb9ca72f066c029e90ffda2eafc3648bbf7ac92a83", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.166.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "500a091747a4dfa5c181250e8d317c1231d27f39cbc6f13a27f8b14f3d3e3b8b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.166.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa6c385076e0ad5a0d8063b93296902fda361662f5f89f95a8fce898acbb0f69", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.166.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "922e53d5513aaff8fe757cce2310c073a096e9be1dd54ce6cb1ce573f5c4527e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.166.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c567f1ab63a243fa1866752f93ff51acb887d46dce6fd06d79dff94fff9baf15", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.166.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7acb4175bbceccca921ddc49781ce785e955cb3733332e9e3b405e7dcda5ca53", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.166.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9bc366989a91693a32cfdfecbdda8d9574781244a547eea239a56d9563f877ea", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d819e1c7cdaedf581c3c92aae36ce76a7eccaf21a5eb8b0c5a444262126ab02a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2bbd30b31010ee8e64722e041210450b93f46dd86175f2697104e5e26db705ff", + "shape": [ + 32, + 16 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9d8f9e6744528760121c40928f9cb546dd3d5fe6adc560fc1162db1631ee025c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "408ea777aa4d029875191b781acc7796f2488ab862ffd7103196ffe1f1221669", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7ccbfcaaca7c7dcd0c4b39329499feeaf8751130cc9f67788133bb6499ea10ba", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8518100bb0363e6e0d3d910353280a5cd6d43efbbb61daee0f56cf437ea084f3", + "shape": [ + 16, + 32 + ], + "shard": "model-00004-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.219.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05123610f446b9d79ac7633041c3c35610ec30c44b58a1138c8a90ea45734487", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.219.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc009c49cca5e52428af64d28e4dce7c3003ef1489b43af3bd1e2a639f151f58", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.219.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "de3aa80702c55539558cda967bdde90b1fd35b80c549aeb47d7338b0a61557ea", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.219.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c14524835417795f53d5c7ec5c746be9b6965e651abedcc310ba69608171fbda", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.219.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3fa7c91eea4c7a3fdf52c7607b19e50c4d5d9bf3a3ec429893b39aac6a0d036b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.219.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "74a125d407f107fd3985fc87bd60eec054dae00068bc6f27c93fbfcc1648235d", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.220.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "da37baffae904e31e2179964de9a710ac35aaa50a5898bce3dd74bb6db54c057", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.220.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5e72e8b6c4b1c0b1f7f4d8473d831a29c094302da2136572b5b80b346a8fea8d", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.220.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2ba52106784ab597141329ad017d52c1a69495bd18195309152feca4224d106", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.220.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2ed0ae78070579444fc55482e8cb0f70618cec5b7ae8af47c112d2b5ec9580d3", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.220.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "74d7794446689d9532d7b07410c04ab508d1e24477b9eab6c8b95df70b96f824", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.220.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b5890c4fc2e68096911d019708debb26fc9176ea1591a536e3f647097552fa3", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.246.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f2063aa1250ecaf4585ddd3506b4a0954e3b176208e702784680da2b95520490", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.246.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2359885911ba322a91e2a9e13f6e15c19af6e65014e647d55fee99251707ddb2", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.246.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "89626cfddd707c23fa18d627bc9b660cf14d5bf184a685594c279f2ac27619c4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.246.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "caa6387d0d58e6b57373fba3eaf75fd71a790fe434e88d895e9b1126c171653b", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.246.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17c8531d0efecfea65496a306a6b7e112b09e0cc6c73017c410c4f24fae21453", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.246.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7999541d055b8620efc8a210341b01311dff7739dd1720611276f29b40dc117e", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.28.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "00b0b56dc665cc3e411d6d9f63e5b9089cecf49b11375396f88c59b26b660f8b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.28.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f33a448755a5b5c3a6089a76d12f4586d2ddfac1737ed62001d8a7b9313ca303", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.28.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c0336832d2a8f193464410f254d8d41b521bbfdbe650bb8078de4d581f1030aa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.28.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "451804f822b7bd350d865694397b808c929ca5f686f6c09e74ada61aa79e4ef8", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.28.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a650623473837cde3ab1a04757d104f5a3cea6adf46ed6fddd7561a830adb20d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.28.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "53be4c5c91dd9f54a85b56df245d3450ffa8663ad725366a926dc7ba15d687e6", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.47.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d6bd3149110bf395f319610ee63c08d391f60ef1289a66058ae77eb4dbcb9255", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.47.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ecb44d48bec37257d3d4e1402dad2dcf9bfad6cc2f0c2b08730b8c522d1a32aa", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.47.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "903f8da572d0d575efb0e6846d61d433d0ebe7cb2009b2479e97e17904d71fe5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.47.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d40b651f8a01ad32040f5faebebcaf76fef7d1fb5a5bfec177cae316184becbe", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.47.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "11b600371f12029daf9c43df66c5466c38ffad3ce3afb6f23e10c05d7c83b260", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.47.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "721a9a5c127b18e069e3f1f8a02259eed765b32d79d68c004bd54c944eef8ce9", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cda469245f1b884e232c983e94cd2564a078ac52f725e0f9ee955fa53878f296", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "04a1c41e8c386c0b59e13e6e59153dd5ca6acd043003d5c97da18b13a701b4e8", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7b561012347f497172847072dddc46863a26211325207ae3360d3ccc91480608", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3aa27c32689e3140c390e440cb135ff8aacf720dbac9d8d7e97d5c65c9c87ba7", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1a578fb83c2bd5f0b58a21c360d42a5a9be6700bbc63db126831e6f5433f6da5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d8d2179fc979cc7c1ce19dc2ab1bbb8afeb01acddca520b119eea8025449f4cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.69.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0a486911385cabeb45b3628bceac9ae3fb0873ae3998d6117c03674ab152d5f5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.69.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c7b598f012736e241df3711bf2c72cf6d6ce6b1f7f27b8a1bbc14c4b03fcfcd", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.69.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5d0b546900dc33c335cc8f75364eb11aa8ea971bafec44d876085206e469b31e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.69.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d90926059985aadc175837a3293f20260776fa71652e9127aa949877c967d009", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.69.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "47e5dc8b8155669a7c2d565932476ddb9ea18d67b5558ed45087242ebd4b74ff", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.69.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2c2fb4f8c0139a80822e21c5c92902d31be9268d119d9b99ad32b462c36e3baf", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.81.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b8e90ccdfc19fdca5cc4109624b39968b3bf5ce0c25960cc41d18db1f1ac168b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.81.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "70a2930ccd49a0868d0f9407595fe0745f97ccbf1601bec12aa1fc7ee0cb1f5d", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.81.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "365048bb4885fdacfff93d8dfe341cf6cd60acb425f7eb6f007220fd3776ea5d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.81.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1aa078b64592bfed50dd015b83d99eb64ef8e4c007e144fdcf0887c21676197b", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.81.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2c4875a761765e60d8eb7ed6ce7c402e7e05b222d336fca5283458935bf890f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.81.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c7086a2f74a3df712af9b99ecbe51751eef159b67311492ba087b281fa5efc53", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.91.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8612cf731eccea45afa6682a70eb5e9da9abb7aa8ee16ed74fbdfd5b42da8ee0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.91.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b84dd30fb998d34e3112d4dfe4827660b8e068617e78669337625c38481f6861", + "shape": [ + 32, + 16 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.91.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f657e5f1895b5498d66eba886de58fe590d2e1942aee31b5ac7a695a0013cb4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.91.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38b103aadd5b12b887120364918a46ddbcde4c129782463655df91077ebdd577", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.experts.91.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dec1bb6f7ef35bd209f370daabfb54227ce4922debe91549e319197296a2ac69", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.11.mlp.experts.91.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9a111711cb95ac5dc1431fe5baf83df049fc23280d0893bee67d6bbb402d08b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.11.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "e2787a7e4040f12da06c6c59c040bbaa3f36839863deae84ea13db4616ece215", + "shape": [ + 288 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.11.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "1abea529ab0e671cf9403bd9094ee7400ba42deb5e93f11d6a819851b3a95a15", + "shape": [ + 288, + 4096 + ], + "shard": "model-00005-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.13.mlp.experts.13.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c93356f742ffc8abfc45db2c4f0916a88bf23b257f39fa989d75ff4e7acdac7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.13.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f75e3d40c1044759e022750411653445a032001558772c76ea1329587439b762", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.13.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "240b42692150d34f082c79502a31bff58cf7d6d682fa01970ef9cbc95bdcb529", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.13.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa98050d91b3b38d4b89f98f76305c2ec65fedeec5e2faff15b09c3b898aaad1", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.13.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eaa0956cee124cc98a1448c4948061249c002c0109eca710c34bf2b203298d3e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.13.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7635d714c651665ef344779d49aba73b5aaa8aeb8fa571fb72f888a7899a91aa", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.142.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "10f407433b16277a12d7b7f7e89ba912dcb1e35e10a2dad6a59b0962e8f4e75a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.142.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e98f315ea8ada08ea92f4be0922d49b1d807b6f366b3ddc48b3d3219dfce4d1c", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.142.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8f71ef2114c92dd1d9d9dbe7777212b69f38d2d0f2be27fd4e2617847c334819", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.142.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "667cd48d9e5a817cfbbccecc6cb74e2d3f84e099dd65aebbdcb722cba292fe09", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.142.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d1c434cd12b43f3af76b78e18e24a3ed816ef882e6cc8e4f7bf7ae078372b1cd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.142.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b226a6fab89d51c2a3e9ef4a9b30fbf3aaf9db94ca0a394fc18de6212e8106ab", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.145.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cea2d2db81610cdf24101ab436e87d0cd43790da74c2104fb2e9ede5ac0894ab", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.145.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "586f1521f87f3f8e7287e25b0a863f2fa4e4a08bb6e873f644687ac072488560", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.145.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4677f5c2ec4d4cced41e2c9ba9f737b33eb3debf821bca63b9d398cac7a37a49", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.145.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f6613f514263e0e087060f5bf5701f044901cd080c06c78c13fe476f72e30d20", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.145.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5cf66e8fbb101fc307ae5dde6bd2c9cff32bb75155058af55353fd06f17214d1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.145.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "59cb5c7983d3e056fcc005997148f4739b55f6446c22c2eb1678bb63d16b59e5", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.156.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "01ec1b9c9a4fbb33511aafc109fe1005236bc82a7602521881426d8489b16748", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.156.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "71f92e5351157181634fc73f6d269d03542083a63a5bec9113ca453bb647fd98", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.156.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3cc89bfe53ad4226f58c458224a6852c91c3d18dcc8b52f0df76704e03a2b447", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.156.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5a67010b25edb97a6a57a02f7609f1a05e2f26f7c00141c03170b0a626bb0cdd", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.156.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0619599937e4485ae80c861796b8eb0b64711904c1748253c77094f0b6eee934", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.156.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "80757d870c569da92760fa371044c7e09658d2ae3feb8054bfed171ae840d97f", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.169.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "85907ace964ccc8f624a9776072c3cb35958e91a61c926e9de90d0e38e54feb3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.169.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a3d89cbf1915bec16e7cac2d19925313f1385d32cb9256b37621c67e0d60648b", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.169.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5c43ac5bc671579ee7398d0cf75ed5dccbfe1a295a7219ccdd6e260f92064f0f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.169.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "332907e4dc08b4c1051d48b426b69ca43739d9b77b5a38ce8a99f6c4b656d4cc", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.169.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "64de3279fb11829f6a43fc6157d9c7811c71c0818cb9f2ffbfa13c274b767b5d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.169.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8d1bef756ed0fee53ee23fbc7e3a4dba841700da31d20775982ee53a91912e8f", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7bc78f0edc4bb1201e000696b48f2525998d517f74b41367f1351f9bef9b2a36", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "285f6b954486ee986a0efbce0b299a5f21259170cb542fafbf156c134ae42e8d", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "892391ece483a767995cb6a4eda5f291535283f1435c210440373ac3b34b091a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c38788abb8ebfd9c53aab5b632af2b173f299280c5f1aab56d22535d18822269", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b37e82cb8dffe4610929ac26a3160dae806bfed7b797c657697ddd8eea9c3e19", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "69d6df5c579624c7e8bac6b4bf36d99a141a119821bfa0c62a51ad3b25120d53", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7f6ce51fb73b8b6d2a3d6504348a43d12ef4cc39d14fef53daabf34a05985a0e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "701a52e2fb20cd7d066a2f46bfe75c201a20c96afe648ec0fccca4af1a34bc4f", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "302fba5ba9a3abeb5ae67632d0582b473f950ebb37f820649b9d3918aa64ae15", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d621e3ae900a73c9ed5820d634267c46ce5a393b5bd334d94eb4e986192bd6d", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "565d187d50f0bdbc051c1c1a0f88b9dc5a6021da909531bbe333b5c59fca43a5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b961d4f92f9ef72a421a0d6cfe9f2badb0ab8cdb19db9b6b58c167259e4a204a", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.187.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8a6f38eae284be7a1e7f08fb481f079f0046d252329297c0160a7887fe36f4f2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.187.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e3164cf832f3691ca60bec51c5c05681e973d8330aedb98dcbb039b772365c49", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.187.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8a5a2e35c22be48aef5c9ef88986ddb00dbb2b4006d721ce98d3775ed94cc12", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.187.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "59f72a2653b8b77de856b3e13c5e8be7212c8205495e1a16b5e151acc6f094d4", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.187.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e2da77cb45a43f58e2da1ddf8b2c129b914fc151d6a7cba413f0f89eab74d7ab", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.187.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7379b193a59900b49f032ebfbfc6d3161460c8fdc620b79dff8c7bdf244065db", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.20.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0fba0d72d1a5162177bc7db6dcdd2ad2fe4161797d1ab1af85aad74f7bc5eb76", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.20.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e94682e97b03a1ad9efb937c04774cffcda6a6fa3185e5d42a38b60ae055935", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.20.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b46f63fc5fec68509d4f5b9076d47a7f2e7cdb8bee2eb48a7acd8597debac424", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.20.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "47728b249e8985c6105c01ac65cf352dd52691006cd74edbf477fef6707465a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.20.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ccab14653be73aeb2baa3e3d7e58db6d4a9ccf1a80b4cc44c55779d7b149181d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.20.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "70f7125615f68af9d4722c9417dbf48e7e804f3aca687aa2da18ef098cf8da72", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.209.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "382909c19a2a68c790e3a93b914cbee42bce2e3dd48ab05259fdcf0fd77623e7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.209.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9fa718358ac158c4340af1620bd2daeaa6ae8f18722795391ecc1aeb644ec80e", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.209.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "185871eaa69e7e2fb39c1d701128c014ce87bbe62175f245b06491ee91db1128", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.209.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "06ddea7287e3bfacea9bf8c1f51c3aa09e095ba98d6c0e56f3c591036de6a161", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.209.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f848bb9b244a471855839f53ab9e6bc99a1fd8699dadd6ec2a70ca178abcb681", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.209.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6f677aa6f60232f9d2285df444f6af85c257e09392c64b683e2bb832eac152a5", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.235.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7cdbc58d269364ad36dc16e008e8cf5d2af1ce3a6c04227ec277dccb302aeccb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.235.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eb0b5e74fff55cec1291ee37a8c8487c8dfe2b14dccf7263376bf8dbbfa75300", + "shape": [ + 32, + 16 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.235.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c9f722ed570df488fadb17e6d4aacb18874cd87c15d06cfc4782a255e21e8600", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.235.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17c7ba29e699a9bd16c6ce5fa7c7ca5969e21310167621ca9759de3a145e3330", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.235.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "532a40e7cd90aaa33d0231ad0705c91fe34bfc4f0ae1650b026019020c874804", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.235.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e3bbb01f18b7a46180446dd5dd6359eb61318913e2f27151af6d886a585461b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00007-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.25.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c60493b9246380f3aee1197a2d5162ccc93128923d53651ab1ccfc90c1620f8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.25.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c5fb0f484db7000ca2e2b73dff4cc678802a89140cb4f4e77eb0f6acb4103c14", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.25.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f51414893bb4ff76dd9c7112065e4c6f8493c0f8787b0040d652f4feea77c47", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.25.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "13066125e0f3061559e0fd6ceeb86cae8fecc769db32e8a578848d7187a46fb4", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.25.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "96d3960967d516e956a34888d8e07b9a4357e0102d62b290ed2d54d7e16797b9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.25.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "925c164d5edfe7819a2a3422a0392788743c99992f736bce6f017617fb2b2c7b", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.52.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d0b8cb0e5cd6fddef60e4ebf8f0b2b13978f04e14e5beda90803a71a23486c00", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.52.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b21b3b31edba192c7a49690782edc2a233f7ae89ec2837bbbc4e9de8a873f09", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.52.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef5015ae0d04c3f920a2de8c3dac4d1ab3f5c4f93bffb71520265cd21af10d0e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.52.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a3c2a95f025b79fb34802869b14b25488d2e8f1b9b3939c43d750728bcc3a4cd", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.52.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e2e3095022e7c7d85234d472d4c6f2abd17fa1d5f26cdb85674a0db5142c44e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.52.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01a759a72677963671727bdf6d5153b3f7e92691b6895dfc2f24788ae1a74ef1", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.70.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "39448fc0b4627edfcd42b98f8de1b842e1cf89b1f0cf2a9acd0128cb1c51a3c8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.70.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9499ce639a99cb70f90b65068acd79729719e0d9bbda368942fc94975bbbddc2", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.70.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ee721465d03b4148f189d328abdbacc2018638f898e360c34519c677fea690bf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.70.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3d78783e48aee90089a9b307fc5ce4c99d4884c3987cde876f1cc2d8aef0d491", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.70.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a1b910538eafed4deff336fa5ca19444a0021d6305cd470f45c0d6b61d8a9038", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.70.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f6b2d2dc55a5f5864bf322ccb29418d049f301eb59dca751eafefd3f1c2d7688", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.75.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "357d83a5b51e13874def932cf9be197e31390b9f77e4112843f76d6e772a4181", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.75.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ca22d6ff08d4f1ca71f0de409b0eb330a5878a514a65e5e48f942a6e38a83c4f", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.75.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "72e05610999e957a07277f087bb57d7e20b4d3bb99e34ecd0bedf0ba70f297f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.75.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c92c79c31f77ca3f9e3c99676ca05999f7d77d3093458cda5f63d2e1d3628546", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.75.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6620e32acd6b5563ac881dd73e5d8f2d3d6f94865ca446846325b763c4bb05f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.75.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "af9c6baf4bbbad3ce50d5b984732cf85f84e77c7e52ec8d35e0b1a547cb1192e", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.99.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99334d978b88f2d984ac23e99d6b3e6b7b91cefd4035d52cb78e8d350ffdb2aa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.99.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b0b64c92a969eb6073e96202ab94902414f543c8433bf835eecb2eef380ad87f", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.99.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a42a0e1c3bb5be589f264b191a7251ba2312de87e88c31456b3915068057c12e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.99.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9aea00bce981e7fbba0492db61ce2701e24f52cec04ab22f1a7096cc73de13c0", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.experts.99.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "09dd048fd8348b0e530027c8f97f92157a75c32ecafdb92271100a9c4924dc3a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.13.mlp.experts.99.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8097e41b0aed7412ce7f455f19938b501a25ce640442b16f04117d3af1230ac7", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.13.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "aa0e653cbcc019d1d33bd1fc69c63666a773ce8551062bf61de9cccb9bb2e8f5", + "shape": [ + 288 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.13.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "5800276df103218e3ac4e9d754deef6123f6c87eafaad9d94af5fe0c04f84ac4", + "shape": [ + 288, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.14.mlp.experts.121.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "38b776fb81660c81e27cb7617d817d99ac15dfadd9e95cdedb0fe533b04605b7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.121.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "13ccca5a5de67ba0e92627be60c11fbc3b3411099f5c9363b2dab4a469959a02", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.121.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e0095a7314a15da7d69d6c4e04c58629eb9424504b1e695617c4c220219cc818", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.121.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0f32f8fedbe5db3a893f667c5a68eea0d16068302c4965404f1a8696c7c5e69d", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.121.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "db92c83a8127d8655e67108f35ebb2ff896a2816536e380b792a6e29fcb5319b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.121.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5713142bcbc66a08a098fc6fb3f14c46b35b62aa0fdbc52fe544367fa42a3678", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.143.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2f2bb63542d740fd3c470f43610033e377c8d886bc3efff668a169e1036b3505", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.143.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df66270d1a014df7ade729a0de35d26288f9062c9c7a25bb7636d0d898145653", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.143.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a979ebc10382c5418730f32b8643c97cb1e2617cc1541872db014d2dca4da15d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.143.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "30c09ba322b759704a003a9d8b4c89cef7995c2689db42d554a0d591f4a680b7", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.143.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2e249839f286a1a42c4f1c2dff12f0500652cf44069be528a098b009f6df5bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.143.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6978e42b3b0c76699a95e687be2ec07da9b6654b93ee3c14e82252a3c9fb751c", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.150.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3bdbf59b9174567c3ebeebc39df0232ee4759d1bb5d3c340f1d0aa7ea6825040", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.150.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9c17ab515a7933f2bd224ba14f76b83b9799b357c605eccd53daa8c9b325e1ea", + "shape": [ + 32, + 16 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.150.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0f945db48907492356df4cd5b719ad6a19fa82171e75760ef6d12d9b706511d0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.150.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c1f2905fb34b4f835b4c4ff0a7fa79c6d278c92d280d4583e0d4b20df0399337", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.150.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e34c93e385de8e21ef41d6c78593641b62d7a235cd0c76186515334c641494ca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.150.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b7a1847c267320de722de1d68f30cbf66f4f13d107c11c77f57be03597eabbb7", + "shape": [ + 16, + 32 + ], + "shard": "model-00008-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.180.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef0c1f21ff9fc702fdb72a6c1f6d73dab2cb0f650fc8d6f6c60e97bd5eb845cb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.180.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e7ba80eaa2cafa64b29a15c5c759f51dbda8e09c921cccf28b9c4e0da6224c38", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.180.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d9650eb488cf5a3267b19510253321a7ad48a919e1716fb779d264d9dc7e6389", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.180.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "27b54945c509dab45ee64a126af80ceb1479f36c05f4e6f8f8dff7a2307addd8", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.180.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2010eef9cf767d644abb5a309491eb65511bf18b573c002efbdc00ede4e595e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.180.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "967df2903123efd2af02bcc108690a2b47eb3d83dd5ecf9e8396951b742557fa", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.205.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fb52da35545eae06ab478ce227f314385d03666fba14d7ef22455848a21e540e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.205.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "054609de4c60367f591a3846d632baa68dc5232114118d46ce84dd6d6fd58743", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.205.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c38533fa02eec3c7ca9b61f2369e17708847d0f6b4351766a0edce93e224cfe3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.205.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f42240d9148334c7faac8712b57a455c0c8c5513c3101921ee6cdc9e08fb12a0", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.205.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e55f738cd247a5421fe344849fab208a2d43c3c18994e2920ad757911e075db8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.205.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5a34e8938bab01ba5b85f3ade63cd7ba3b300f47c2e59d3989d35d5a6dde85a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.228.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99155e38d9a10989e4c3104daa7f93eb198ff00840822e7e6ab5eb9e32e6c48b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.228.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5a24e64fbe7cbaef6565586aece4a61ad917faaaea242185a4ce205cff96d68d", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.228.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc3b257bf3196040059e39f8b66f5ace2e97ac5252d0fa24fc42317d229beb36", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.228.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "333044bce7c78bd126ce450f8853fe86eb51fa1acae4ce6575c1e81f9fbfcc26", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.228.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05dc8b6b512f9e92754b3c9eab73f2e46d0a114e6acfa4196eeaa6512ee7b887", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.228.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "11af254b0071f668cc6ea21fcb4c5d46bf9cfef8134919561cb5989e0130e80c", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.23.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d423a0d4af5d7df338c87be0ee99284f95a974e624cfee6ff3448027483e1e28", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.23.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "687c029dfd4f920903d5fe273bea16e8922374db528a10a2223aebee1f74f9e6", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.23.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5eba99bc539d428d4b88a446b7e036a750f19eaea5e02a99814055ea5d47f13d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.23.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6c7f3d53caca7bb4f31c1c851e7719fed41f0a5b6cf0f3962532c1fbebed31f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.23.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ba79532189fa7e3662526aced540fb6c0f08d8f460f70dbb0d16fbadf24f554", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.23.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "20f1dd4dd342437b257c8084ff16120c8bfaa761e2545ef438b61fb1923c5580", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.243.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c900f06a57c33d433619217832823e984810415297f955fd6fe6c2062fdfeaee", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.243.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aab24c610dd0026d26055d849f9d0fbfa4fcac3f761fa0770f7204477a83d56e", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.243.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "88efc2d3c71a5da2cf97c753a76005c0742cb95ca024c171f3660d935581c321", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.243.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8f983c1d0ca5ec89863e12130fee50f42e8897f3c7f3742ae115bf3a3051173e", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.243.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6b9c59b3955a6b292472b98b5fbe141c2553f818087e10ce410092ebbcb64072", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.243.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f30f322bd92c14af8c476d29dabec91b820872606afcefadeb4ec385fef49d1d", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.275.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "34e7e1f8595ee71dbd50f54e76f752ae5cd1c0804d443ed1c6abb85445c3c09d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.275.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a5ce3496a6c83043b4198d4c4f41d1f656281697ed86ec2ca653e7a8ec746bac", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.275.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31a1c19688ea33324a067c0e48200050692fc99da2a44246299421ab6cb6a128", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.275.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4b83a80710cea021a4edbee2fd880d41ecbe3f1d139ac83046a109769bff8873", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.275.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4774d42905044ce66ddcf4f6c6e29970b72179d1c14ff75a53cabae442aa5c1a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.275.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7a155347efb24729f367f8ed171efc93227ae0ca28b2629b2197d4b3910c5e56", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.277.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2cad7e75730a4e7fd1cdcea445086e9c8fefa5c1f69bb22c54901470d53f34f3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.277.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d904c1229cb202b7d69caf351f7f77f08ac150aa7f636d0a58ac88f96f47319a", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.277.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ab46ef9563a2d3cf15ca051ea233c932fbe91717bea7a8961d6190dc90833113", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.277.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "57cdf3c0493bd857e6800ee97e14849ec49b9dc5bf2dbde1112f3f25befd5aeb", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.277.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "70e9d8a88cfe22f3076f20083687f316d019ee798ca6c5a98ba2e21bd410731c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.277.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1d3b48b00b436de1d282f3f46e90f46cfae8dc5425374c850f14069a12a03b07", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.28.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eb9035eca8c90d5c866034bfd958b957784521769cfcd781a8b7d18af1abcc2c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.28.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d7c3818b96cfbee479095a5ddf215db2806a5e0326784e26db3d367748491478", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.28.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7f175964029d5189f5ddedd58f612ec4bc87570f791fe9b9f860fce1311625a3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.28.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "75d3729e4d455bed891471d1d1d30d36c6b4ed305773b7b98a013f1df25706c1", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.28.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5d8f718c075ae6ddf9c27109907143e66cf022459f9d18cb7bf060cfb54de541", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.28.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "41a1db347fe4a8f48be61edaa5fa437ff6e97d26a6c203d25d1cabd3fa9fb2ae", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.285.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d5d6e4bec26791d5f7b7aa7ee547f2f5ac7918d30fa9f416f77388bc78a6fcc4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.285.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6222f7c62f259d772ce45877430fdfb494f0c54f79933345de6e477e59c5344c", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.285.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8bdb08853e8cac6651740ae47f137789bd381a82d61a5cdf20ab867594677614", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.285.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b432d59efd2b59b8902207243109d5125a382a219059e2df3fa3cecc9809578b", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.285.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "700a5ceaa31395f15865823fd5836e0aabd8000b952b315146a2ccdc593ddb86", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.285.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb5bb067442649b83b8e798e3227897c48e26c59cc1a656d49e4a48764dae5ea", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.46.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cb9d830ad4cafe10fbe0a7187d04f9b57682ba1106df84d09b156fa036531469", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.46.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "26093630a6eb38f35a3fc5bd24afbe47fa1c358c9aa90ee9d119caf105b26823", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.46.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "117c2075fa4976b51a985d5bb570cf1aad30ef8dde369efbb4c011fb87901962", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.46.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17ec7d4964baa4c49ea11a0dc0cc775b1f14622223d5fce1822b21ce704745e8", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.46.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5e214417f5cb2c3e2e3d2e650b5ff38070f1384a19c33b8a4fd1963d9c176226", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.46.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aed4e89f5bd739f545ebf10efa1bad0b828493ce4917f631a755a97f780d6a27", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.70.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1929e483d4380c075eca7c5a3a449026db4cbc4f8d854f7e99264ef5e17f410c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.70.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6fb720a1048814db777c0c732069c274cbe98bf6bf189d400c9bacd50cd5c71d", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.70.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b2087c2ca78d564b88a9cfc93e76f8ac510c710d30f1f0433a2f7eeb29b3769a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.70.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e73aaac1aaf70cb9e7d54ae5536986174147344dafc818b41aea01ec81405179", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.70.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "80c5b14a6d3aed871033c1ab84d485ed4737b0abc883068d60599675f229dd84", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.70.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f13042d6f3279c8ade56cecea291fb37e86f005f1c2c0c152c466cd251856fc1", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.72.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ae14f7bbf6928e90fc2583772ec51731a8bd73a89e82402e0962cb0cc7c5e82", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.72.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "85d0adec2a056135cf66faa2a92a182d74f3357f21339f3536ea7efa052938d5", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.72.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4118e22e193014b032ea1c94a9cc0ada3f2af8980331098ba20ec69a607e890d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.72.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6c50c2ade68b4293334e37a6643d7c650358340515d8922bd2f0b1588eb2b192", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.72.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d1b217841ad0c7011aec78e16a3d257e0788a0c1e8ae9c00e3f56fe025e3909e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.72.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "092594df3544a0624d7c796fe3616fea854af745dd07896f886b151751352187", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.86.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8344730ae8e05b7172a1c4d90e162ebc46a8de7423af485fbe758d9783fd9768", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.86.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ad1c39cda56fb337d10d42f9d2b744826981793765312ee5641b205d40f3d198", + "shape": [ + 32, + 16 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.86.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7c1330bf5964e2d613d6ba300e98ed72977c59a286bf193e3b02a129859c95ea", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.86.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e759cd98a461f0978c20fc669530a24882005b92b50f0fd694845fe179103a6f", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.experts.86.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d5e2b1d34f1e1a333a215e9748989e4992f84059aeb2dcd07f12acd917edc89c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.14.mlp.experts.86.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e18543d016537536c90115a5e37815010f08ed1a53adbfeaa562526ebb07bad", + "shape": [ + 16, + 32 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.14.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "207144206bbd46e4d51f018383e9b8fd1b3bcabecf8a1c6245ab37b3931ba6fb", + "shape": [ + 288 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.14.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "7eb87828f82f679850fadb4f8ac9bb0e2fd6f9363d623ca0197a323adefd784e", + "shape": [ + 288, + 4096 + ], + "shard": "model-00009-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.16.mlp.experts.109.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e7f7cbccdc6ac48aae4525119a53ccccc09e869534d8ba1bd5c333d06e322e5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.109.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b8cdae3523208db84bd4a8c9f7e9dd2272ab046ee70a6e9067a166686c5c39a5", + "shape": [ + 32, + 16 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.109.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1d5b1a876d2e27f42e315d7f5ec502a4dfe920a75e73a6510edc0e4044a56e35", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.109.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e13be0f20014065088b65a615d5856ecea061af043ae92eafc574482c84bd8ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.109.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4cabd147d3193092a6eb787574f91c2a399ed721a5e90d5859553f42e7b681cf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.109.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9271ff4b86ce80d571b655ae1d6cb3400eddcd8b4f475e1cb562c3d06af2b6e5", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.136.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b755fd45d55854e600261287aed115dbd086f06c3de5e0d74f9e5459dfd104a8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.136.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a62f8f740c231a4521b59872ff792c3fe691b0f1466022e8da0002740ee82315", + "shape": [ + 32, + 16 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.136.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0caa8b278eff521cb8bb70641fab653c142122e8c12ba2cd648b2d61becb5158", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.136.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "569a90166db506251708898e554ae1f1983da56cef23a0b212c20492ed70ada4", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.136.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3a7fe2b2d3fd088567e5cb6be4f10b0df3c0d98b1543d99b17a61152503909a8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.136.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2bc24a3a60ea2edc713a4bfb74f75a1f73bb7073c0aae7a78e86d01b8965eb68", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.185.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5e38eea0d5549720ab5374d2a9725927696b3d33888f46ee24229f46a470adb0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.185.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "655618dfdf76ccdccdae135f0052b84c1f8be8d4a9a203f4ffa4a4a7bf4dd60e", + "shape": [ + 32, + 16 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.185.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe2e932d4e847dbbbc77fc05df0164924bb7c296b1cff5bbe4b040a42f066cfd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.185.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b4f4599817342089f154e55f3836e08f7c4d51cc1c83e8a1e2a6ae0a2fcba8e0", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.185.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe6ca7adcf6b654f9b4a752c56a82c3b128903afd6e08eb122e793b870049c01", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.185.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "370e1123700191dfdfc74fc841bd75ffd450a01c6b789a28b198f058647f4dcf", + "shape": [ + 16, + 32 + ], + "shard": "model-00011-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.219.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cc911360e662c80fefda423a93bfbabb2932829a95d1cfd01b041639edacb6e1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.219.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "98b02e59aeb103ec276092796008f88f78bf775352b907065545085bf0be1bd7", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.219.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "992d1e28af596b4d7e5ecf50d45d940156ee87598bae0d6c298f4aab828effd0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.219.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b40f2c535f79ef7f9338810bbf13d5c3d3fa08619f08823b2c13bf5189831d0d", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.219.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9162dc57b2a939dcb30cc0537e51687198367b24a73b9735de1d4a5c4d110d16", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.219.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "21365e9b4ff4230b5678e5dee45e419e27a4ad04259594f5447a538e645eef10", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.224.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "27160ec516a3c4b41575b9553da7d6dd1f01935577b102cbbcd5a2737b7d514f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.224.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dee1a6fa987ee0347b166a4b6e9ea930b7edf81cc2f8906df3182b07bd39962e", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.224.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c75229f5ffe942feabb2162b381e0e74fdb954f523f5d08bca2edc55fb8a36f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.224.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "da6de91e7b4b2834c7bbf897314c890d16119d4452c977c60706f99fc37e5eb7", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.224.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "33b9eaf4ee2188c7f3d4c31ea599ee98a631a9fde20b1c3b09ff8439912604c8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.224.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6676e46e66973d5bda32a743a58ceb9ce410e263da1d32a299d8eb29d0f46c88", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.226.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7b45c4fc72ed0c42af148b76f986ae80b090ac18437bc601dc6cb07d5c7176cd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.226.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "838a1fa718831601c052790c330ce705d87db4df29b2bd7d3733873b5465d8bc", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.226.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "511698840f79b6b79c58d38e033d2804f2fbeb2d97e097cc83f0d7e3149d7912", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.226.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "779a1796c0bacbd6196e98b6ca80dc53ffbe1a00ed05e21745861f374e411c29", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.226.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f3ebcdc6932d638d272efa40932569b344b030ffa3c80821ff4b57276fc98912", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.226.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "098940368c04943db23c4d6df53c857182f88698503aa24f10379eb90781b845", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.229.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ab659d82563e3787ded5a0847064c966a381611ccdf6df40d9cce9bfb54ef6b2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.229.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6f195233c5ae25491698ab2d7f8d087062c4c9f2d3916c005c5e6835038823af", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.229.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "709b4a43c151058e093c08cf708205330711435c46296d48b00a833c064fd953", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.229.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2a4a7f5a22706a555b77be260095f4514e8a0afc3f11c9cea37f00a78e126d62", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.229.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0af3b10e766ab049cacd6f0a43e89359ac5a3891168c17725d853128a0996f45", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.229.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7b862fcbabc1fc80a78359ca33e8e65dad281b07bf4c02ac19e5680a07ec2d5e", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.23.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1e42d1b38d14bde3f94cebee1f5ada2affa87c4999de70aec55b0000fd5f23ac", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.23.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5bfb938c08f946580e8cbdeaccc812e2c450622414c0a72a5fdd198826f40c03", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.23.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "caee1d8ee0078cdd2c3a67f5791cf014a6a60e4930674d8ce8305f7b936a6a67", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.23.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dc23819aa0a709ba02e3677eb12305c31aa428d168c1bfb6385ac61b3cd33c3f", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.23.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b4ebc885ed414b878b9d0653dec5e2a2a113a5fc54e5313ef47a66ab25406117", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.23.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01fe053bc16cb33368504e090bd486b22c6ce95c17f2d32c95fc93b0cdfb896e", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.237.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "679aad92ab9aeb25a3585651adebd9f0884bb2de73bf00b6c7dc691131773045", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.237.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7bc844259b2686e54569b129a9a9322301a1849433cfe87872b064b2522140a6", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.237.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e6eee5e96239aeb02fdd991630d11e4542ced249e1e6b67259724dbf7dc21a6f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.237.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "77037b9b135fe8227e439415e2be74f4a69236909a0403ae8407045c032ed111", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.237.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "751d8c1ce2d1345b1afdb99f494c4da2f6436ceee389c4833d3b8210a9294a48", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.237.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "82dcddb17a51dabd82b51607bbe5b3baaf252c173b1f9c31c3d9fa7bfd0053d2", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.244.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "94f6b4447332c492b20163ab2399900c181f9957f1b90ca4660afc4237faa0c0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.244.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cda302fc3d532ae0d0751ada7838c3d953458ad9afd2b79b5ecf1f96f3496d71", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.244.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "813c9d4c8ae1335a2485b69654ccfe21e121c350da6fa8283bbad4cde35b3ede", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.244.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "593cda8aebed9cf0670cd624f239639f2279d8cfaa5d91abc464a190cb10a9af", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.244.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1a091e2d0fb49a20825fd078866751e409acf6912dc974697d4f7d51f09f06a1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.244.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "46b2d12eb29e283509247db6f9b013f4b030605724849b07f17c66567266a33a", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.273.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b2a51445ff64bcb01e3ce1a4bbd8db2a8847f002d427754bd239d751526a1c95", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.273.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5012c32458819af66173fc3b2d445c36c6a8f7eaca4d6870b72822258c913b74", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.273.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "88be7e5776b895cb612716e7f8e13758a2d4760c7d27e277684097e9961ab221", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.273.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f1c9534f254c389b18e7513fdd4f6384ef0436b3c8e6ca1340df43ce57d5beae", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.273.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b83888fa3a8731a614619a15505ee1e1f2a15efa725fd5f362c16cb6ea62d904", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.273.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2581461c751a554ba0d27ca0f1e62d5dac3634b47cc47cf69cd002bdd5603b8a", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.64.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e91676e3f12684e8806ec6cc70ee8067c11a792b5f2cedaca64b04991365a4d4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.64.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4e1e411fdeb428df50a3fa7c7bb73ee1909a34a108633e2bd31028518d4a1a4", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.64.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84be974da06dbd3c0e656e487320b1aeb4b8c0e8232adf2f6d2bb53ff24249ad", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.64.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9c3c30735698413607e3bacfe8a29606eff6697fbb4417e15bda51e2d7f6d8c0", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.64.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f165cacc769e49df11407be416797a077510820f633a58d31fe39cb1bc68e718", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.64.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1cd6dc89e7e744bf5881bceefc5ae113e87b1165f00b2ac5f5364985482ac619", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fdf732383eabcc5e8961f396bdfb107259c8bf546f4a17ac9623cb22bdbbedee", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8eebde27cfafbe5403e171c6f8621cb9dd337d380205edb3ef4840121438d672", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2154d4b5f809c33c94198063812c7ff326707a62cde0c1f4e0c832c34df0e045", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24b64df323617104421c7a56d4a47f89f884b2b4107df407ebcdb29f6d1f0c3d", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e7a276589c20c28dc763e11dfe87dcb1f0673eb22cf92cb4a192f2eef4b13fa2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "83d99f6db684b841656150ad631e96ea191d3408fb6e3922361557f24ea8237b", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.77.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a029b3964b3bc256a1a3c44c605623e439c45bb8019e5b8610de6418cd07b4d7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.77.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "057d71fef29371f478bc0e12d150ddf3d15b08181fecc8665a3372536d94e100", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.77.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "948f76206d32bcdd2303a3721bc9dc54dd0eb7f6ed5f64cc0ce29586b1b70b5b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.77.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "771aa41db2c4f89b4fa51330b857c535df24620d7519a3167f89e2154b0ce4c7", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.77.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "192ad3dda0515d1c6de046253920cfe24ca165151a546b14252c5651270d18fd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.77.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ce1dcc019e507490d7d12a17ff23fd8c152cd3c882b7e1b742f1202d0cd34030", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.90.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d496e7a154409fa81f84a40a6579c225952eba00613d67b7ec91c1787694a1cf", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.90.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c58470fd8d13d1487f6b8d59e9a1704d30172d7090876d9a4fb48d524eed91a", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.90.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ec8720f73492e0be4fbede72a40f5b7fd94e3f13cf7b140489c46cdc5556330b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.90.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "006173bd6d8460ecba03b309ba1354e767ac12bcd56e8ecffdc6b69a7f74dd6c", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.90.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9d65b48ca2f6db1a331103e63f660399367f057d5b8b82fce2a04c01b86a3bee", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.90.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd5dc161e037a516986859c8b99f08a60056a6f65ea8bc73db14c5ab2fbc1823", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.96.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7ef9e809085a8c7651b451417148238d871f274744edf184721305f4385ba0d4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.96.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "39c4a473e9af433422546d4d0c7d1785ec6d050ebde78f1e69a8896713a33d5b", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.96.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bb041d8487e3b634a65abc0572f2aa9cdc1544d1f3040b920577990db7a14108", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.96.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b34a2583d9e56b58c824e6502fc9ae08f668da38ab1778835b257e2fb2c36028", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.experts.96.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "20749b26d450d161d7aaaf309aaa817750afef3e7620adb838d12e8db101d442", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.16.mlp.experts.96.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "acb5511f4bc29537b5e6911d2d95c6fd34d47a80af304a70f06c963e3c2b5a1d", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.16.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "53f3703c12def5a89a5709b4b1ff7210e60fdc6c7e76ff6398ad86ea44639d17", + "shape": [ + 288 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.16.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "0d9d255cb4b1afb9254654f6228b9808a192a9b6322e3bbfaca5a39c08873fd4", + "shape": [ + 288, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.17.mlp.experts.102.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7d8ed4e84a36da954de3ce2118dfc9103c5019e5520251ec979451e8de7cce93", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.102.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5ad67aeffb0f0acaa06aa7edc35aea964ba13105c4aa5dd8c2becb0269e47e4a", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.102.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b7b918e87de43387ff3dcf4806355e5afdd64abddeee611a5b879d731a6e6cbe", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.102.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "718b6eba881d2701e1ae7e7208482007876315addc7b25193332ab02a29b0cbf", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.102.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2c4de757e7421bae19ef6bca1988ea49cc9d1506604909a40a2cf40ac659865", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.102.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fad629df5989d76182c9dcd8d5f3e0d382b1afe498844855a7bed299ed68dc5c", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.114.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e21562582cbddce2f89c1b71c6bc44f578fffd181b2667acc80bb474f2e9a2d3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.114.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "740e9fe235c92c85f0f1b4c6eb6643d8591e64ec53ca549108b2656b7aff36d8", + "shape": [ + 32, + 16 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.114.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8af060d1f0a89c38764758bbe81ee987983219c8d17b1aba10d979540e6fb88c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.114.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "292d24c6ff995ed251708c68e86bf3a3326bb04b54cac7af37bff645f1dba75f", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.114.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e9f3a22dffa40ac870063fdaaf2bb80102d1319a80fac0ec93a50d08325d9274", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.114.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4167a360f4f7c00f6869fdcdc6e28cb9d85265e000a559b76cf9c6cb4fb9aff5", + "shape": [ + 16, + 32 + ], + "shard": "model-00012-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.150.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b2cdb08de11a3e7eaf03373e0ef6902482725cf1f895b7e8852432b6309335c7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.150.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cf90b5080ca32b6ca11d20b72455b2ea730b81bba685240444298e5a2fb75859", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.150.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8294d82cf3bce91ee158e04ae0d5b8c24ad0d5a70f1fc9143241ccfca27d0251", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.150.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c4d805c7a09c0f6eff4f3e360f9ea20ffd0a811c95a33dbf9c93e68a16ded5c", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.150.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2f93c1a904bfe7fe4d84bdd3c5a1d4042521e1ca0ee62e78dad4a8452fbdaf5d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.150.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ba12b51d1a90d51365cd3cf15fa4cc28756852a52e6009748250de428dd358de", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.157.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "58df55dc83b3194c8371ebdca42417ca1c6c8198054b4e834507922d71d10d26", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.157.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f556ccf87a85df6a30f857a06e575dc49261aa7a77d0ea3dab686a3cad765477", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.157.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "15b5595d54918b7d78517c3452f37c88424be65696286f0988ab1a05b9814eb2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.157.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bfa88999b3a245c78959ef4901268c40c212119bd996ad3588470fdb7c782246", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.157.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "244c415c0ee34aaf350bf16bfdcde6c56354e4185d043b14b1da43c9cee57d24", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.157.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4b53957a623029d7e2c608968187040eedae847505ea88942f904cade20d8b9", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.172.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9989d11b74b4fa0d98494ed3bc503af6900457186fb57acb07cd6ae7dbc6db0d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.172.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "65bf4535e068420e38bdd7a81418dd7421467d61d081d3bc067dcea32b2e48fc", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.172.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "16b55e83aebef784c2bbd41e2082642162a317ae2cbb96017176da75e5d5bb49", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.172.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8be007e48e4404618146419d10072eae55868999b8837ec78e51bdb6dc55c4bd", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.172.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9dbb339f57de9e960b561734f8b3060e0f4472dccdcde363922aaf2048eb0dd5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.172.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4d643f4a187b0563771016fb3d3eb2aa6974880e821ecbc8af1fce329ac8fa76", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.175.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fd861f7d38aa4dce478085e89314418210cbc10d0b8459f0ac551e34f493cec4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.175.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a2e190abf0e36d5c3584bd3c696e2c89f6abd1ff8fc18ef7369814a71a8dec91", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.175.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "79267b843b1a6ad2ad7753b50e303c0b5b4c4903904f26185263ce0f48486643", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.175.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4158344fcb8846cbdba72284ab40ed1af6a9905788a1cf5cd638329d4788b17a", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.175.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ded6654b097c1a2a46822bc3477384a1d7df23aacd134f7b8e805908bc562c41", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.175.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e76d6416727cd8bfd80c3810f0502544719d1c178c8bab618309144a125ff99e", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.180.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "695a83d3247041164eb03b2b8f518ecfb6537a795f0eb8bb7a5ab29e2d76915e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.180.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "28e6fca9be16f44d09f6dc74b8be26da83e3198db84cf565985df698bd3df4db", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.180.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "53af5a328ef2bf699b5547f9fd201f9ef84f0154f71cee97bba2096372610e83", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.180.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7fe1e65e1ce3fd4efdc192e4abfa83e2d1864fc6ae0f7af609562a563f4894ac", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.180.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df0963fcbe6cb80114640bf218364489789e2d4cf427496366d6a96b42c08c0d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.180.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d3f6bb32ddcfe01bac3a9ea0085080c11213a91468005f077d70b17312d8ac1", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.186.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a3046dd08a5c673ce82e47f15d581b1a2f25a6ab2583cfa734f614974b2f935e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.186.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de75765b3090d683f1edaccf2b13d60b0adf22ef28679a50a0d25eb79334bab4", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.186.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dfea69b1509a5a1f4abfa6fe8db4667d71521fbeb54197522206e5ee2d7d4d6e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.186.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "847215cb66da3b634224438c0a5984e313c12185e102b32e8a84f56676eb90be", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.186.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ad8285527326f746bf2abd0cfafa14491cb50b6c5afaffeded1606ab4ea027a5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.186.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "370a20a5000fef3b3e50f1d07def8b9cdcb5b2482f61f4ef9831a22616200dbe", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.210.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5dd80384233980b03642685f2107eecb4a8aab5dcb94cdedce3c3eef0af3c4bb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.210.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "87db4105ec8bfc3e73f9c60783f37551f126c63d4b99232a67926d4ec66bca32", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.210.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "252b80269d1999c7bc9ddf50e598c52d44351687dd4de96bd443a1a215a7b98f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.210.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "175e9126ddaf755c82a2543f5271927a9dd7ba855d219904c2f24d202aaaa6fa", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.210.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "118f591f2bc41ddb5be538e9513322268deb98759e335ebdfa756549d93a5f73", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.210.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8326de5281555925ddf195ee6d45f5d8baffdd90cf84ed742c9ec5ccce513260", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.217.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1d70d1af76b06cbbc1f443814d92038f234f6dd7d57ffccaa9ca820c8f670b9f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.217.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "47d8897ee08de0788d64dd9153f244a7c8c3360ba8a953a6c3483dfc4d4aa707", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.217.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "043fc31eb39cb3396a81ceaf4b7ef6b5ce7525c4178cc5d197daa68b55e4f36b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.217.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "da2ff1e99d8ae4121bb51ff833a89ac9b3d285cc97547d1e25cd3dd995511da7", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.217.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2146cb12a1cfb15e645b061d25310663017e00a6d2aa0334f72d3410216ba276", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.217.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0c8b0ea0a4f2b24e8bdbbcc6712af877e001a4cf8bf32309056879880bd4ab64", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.279.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "62a6e0d76690fa4e77096831740c604e64aacd0efc5a0613a2412b999552acc3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.279.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c699cba03cac73250912fbf1fe07115606f2c2b22d2db60a2309f96ff890095e", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.279.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9b7ca8a8c34b9cb349f9e8fff02ba4c13ae4ca2f64c2229f80d7b35edaa314fa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.279.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "43465d005b9175b8a934bf3b8cb1df439a7d395069bd705dea7558a52c2c8690", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.279.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d70439aa211144d2be2e3a507a0e4b6f3c139ea158da5a206736464fbf8d9c64", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.279.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cbd21a9c99846b8b714aa0e8563ca9776937f209d1ebd73479cc9022220d12fe", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.283.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "afb65e192a28c17d74f630689e63da94b26bfd567414e3aad95227e73942d14f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.283.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8600ce5ef19677eeefa244cdadef40d89acaa0d6ee252ec78b77a222bb09191a", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.283.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b634c6d6ca21a05246a4c3a817d522f01ec6087de22e49a35c1ce6add344838c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.283.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "122d46687ed3d498f55c802efc2445b9a7f6ebe7c91b1ff5908b143b56648435", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.283.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e6ce52560edffb220aeb5971aabb64fcb2efe1872e0d8d5a2faf35fc67977fc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.283.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5f7368e45162784495f4c87668ab6bc1cbfda2fb403ae58fd757c8cda0d467a8", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.32.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e83bafdf19c2b555594c60f57df8548239f7b8787938676ea367fae9eda71f9a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.32.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "577a5ed75065b522aa6bc6ff7f555b14c9096a0af830ee30040fa6e520fd3790", + "shape": [ + 32, + 16 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.32.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "24417596ef0474621be7c936946128273892c8798de87787195e3995651aaf2d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.32.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "920586ebab15e7c2eef4b93307c6d4819b3e9a11b47184680ad15540a32c285d", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.32.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b3ce1729f27e4f8b4d55f0139ee2b435ccf8d657caf5bb3d4a47226d30321d85", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.32.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "157dabbf9a00b7775a3267d191b73a11a3bd0130358a8730bf1040bd38798294", + "shape": [ + 16, + 32 + ], + "shard": "model-00013-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "394d8aa1b12b5c1488589b1108c6cf8f79ded24dc3c3377daf20bc6176f9166a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4458888b8a54e6a62efabd745962c64a00593c58cae17d81d089d849d132c4cf", + "shape": [ + 32, + 16 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "60043e98eba3fcbcfd05b8c974d83a6cdf05f8008ec9ba9b2faca2798b807fcb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "718b5605141c86a6985d789369c6461be05eb4ef774ebf92233dddabe18e7df7", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f9ae86fd35b8121cd053dc952b5844144a66cab7cadd9f74364c5b56a5f9734", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1909458e86b85f7f23d1809b70e7936d31ba9c51484c615da84aaadf2738638b", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32866c0757922f2d4533e3c21a1e7653c5277a1d90d2cdba9fd9a068d8e073d1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ed4c6f8bb032e3f7d7be2c74f2596cb60c1a8f5d61f35cc6330bf7c9c3fe79c8", + "shape": [ + 32, + 16 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b364a5042f5547fbf4c77834d4afdf444e12e791b82593d4f6d28b06a69ee798", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d3a8a62b4839e1cdf2c4d416c1fc3cba505d42131dcc6976cd313e4b2861278d", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a666f526d0a7443d9988e6ed1a410361bc707c154ca042e46e5baf2eb96273a0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7bf658d8f18b4a6f2e2fed7770bda2fed13cbf84e06e890366d5f1dee0c2c978", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.9.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "45f7ef137d615a4912a99823a12feb24d2d4961316dc95b217ec6e78d91adf40", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.9.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ebd03469b32469705e88705ac8954d88d10a73ed91addfdf78c5124fdfcf155", + "shape": [ + 32, + 16 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.9.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7530be990b7ce231f81e571ce4ec09d7ee6b4cd724e5af552ef36f3ecc33bb87", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.9.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a7bfdc4dffeb493545ccaffb1f3fb00445336782424ef04cb8df445519467cd0", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.experts.9.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "879016e49b00e0bbdfcb73aa81586947d41a99defa24d94216b29a019268c1b2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.17.mlp.experts.9.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8881e0bc3d91e974a3919288f055ff8edf4acf71f39fcb5b6a206331b6c7a76c", + "shape": [ + 16, + 32 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.17.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "00dea1a7f9320c59a367d84805ffba137b448d26a0bf3437a6e5d9cf7f55e3bb", + "shape": [ + 288 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.17.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "638767dca72b6c4a2b7f2c56278069e6f4e224f5fe3b9c17af9fd1087ecf98f2", + "shape": [ + 288, + 4096 + ], + "shard": "model-00014-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.19.mlp.experts.111.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7bbcf7cd04719f671d7cf7a86049413bfe016c3f7bfbac11193b429026ebaff8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.111.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec967defcf8b594d49b388c3855cb7787312b6ac00d14466d9fa32404dafb8af", + "shape": [ + 32, + 16 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.111.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "09eecdd98f8d695a52d0c8201bd57107af260a53b150eafd4289f78d4fdcb9a9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.111.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9dac0d3135e2b622959ca178464be09510250ba5da7431db92075f977b02b2f7", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.111.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fa668df2e2aed02bea11b7bd5c73d98eb5c1bd78c113137c0c5abe36d685a20d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.111.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6cab3cad3125b1e24a9caa6ea40db2252352af726c46595551e2d399b9fe15c", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.136.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e1cbee4b868a1250b7600b58c11930f6f2b88eb335e46267a53627909505169c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.136.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b9319f4975117239502189823706d58d94d26436944f2dd8b27b907445bd768f", + "shape": [ + 32, + 16 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.136.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "740d00304d50edd24415f66d4b5ae7094da2e1da2360a8b845c465ed66c82e21", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.136.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "341db095c6114b66f6813678e91b6b593ac45303347bdb21aee44b75f3b5e9c6", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.136.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "becbd28f0647f7162e4400d0df38883dc27e73a3b6d74f2b4a1d1d4971bf0041", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.136.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ea08743b86c6ffa38f5a7a50d26e977b183cbe904a5959f7e788d67fe3226d8c", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.157.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3f2a897d94a9802ecd8f4d213ddf10567bdace8717bf7104d1d1dcae23ac2691", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.157.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "627eeadfd792e7de9df2548092472952167a0c850551dc8531089c66986f20f9", + "shape": [ + 32, + 16 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.157.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "324b2021d3f16400047c03a1f146cbf29a302bbef90fdd7fc89360dca39445cd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.157.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5dba32c36af353b9195516215c0b35f6b1616b4740a00e7fd6bb0c319ddc3e24", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.157.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2e8ca19ba1f5d4b4dd7b0434ed7ef9437b56a7088fd0a0936eddee115efa3315", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.157.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aaa58c141d0a3267314fe334b0fbfc386966d8d5b96beaf594bc5b6c7eb443e9", + "shape": [ + 16, + 32 + ], + "shard": "model-00015-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.21.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce8fac4f9918e5fc700c30f4ee5fc25148885b1a2ad36f97bbb50def6804c0d4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.21.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b551203bd6f767e2004e125c2d0a5ac2ff5b3881a1ee663bf40f0d116a3ef151", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.21.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b4b1d9bf5868dd0d6664263a1b5d4e292a21fd2797d5fcadb9e7d2649e6dcac8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.21.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "391aef197eafcad6e67b3b726ede5332bc94e0c78704a8a1d0cff8b58a349749", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.21.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f148901ff7060a1f97c5a64d51e5486b049e3e901807121e4d796299492366f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.21.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8ba3763ee19037fc0dbe3259c161aef490789d5006fcb1150ddb24fe03e60551", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.228.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "51ff2d44ef4bb843e2b1252ef26b842a329f6cd6c771be8df3e405bb3cf735f8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.228.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "714432f7366f8e2c5f1303d269e3b5c5bb109c5208559a6b06f1bb4502edfbe9", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.228.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3c29ba9b6e33272fa552cd11869ed331397d63a08f74b0ac9012a62d51deb043", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.228.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4c3863a21398f8a4c5f49c989bc8b7beea83f7fd51574829d04b2f42ef048562", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.228.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b8885435771a8b8019dce4561fdb690aeeb691ceb9d30ce10df670ddeb11646d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.228.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cab7d633b2cdb4e041b21ef28b4351fda83f83b49494927c3ec8c309a99e2e6c", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.23.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4351660556c393a58447d9c0e4320479b18043a06790cbab5ac17fd5a1bd6d95", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.23.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1eb01542bb36de4a8cf758cba5c8621b8da885191ac1c369613c5668872ebbb4", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.23.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e20f13605dde2d98c5de8ebc9af1246c0b994bcd52a33b3f8990004466a38a03", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.23.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3821955e2b28037f98507e0a9ec6e749607064e6a79f13c0d2a6801cb22f1202", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.23.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6fdb0c2730c4df3d42a4529a6c1bacd6bdbc8e8e89a4e4884b2a27986cea1b79", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.23.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ef607f253c77146288286c91b4edd2e14cb4813c0e174a90e1e392763270b650", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.261.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "56ebb76add5dbf7cf545552a94ac125bdd8a474152aba54ef7f4ed22f2182a63", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.261.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8db05edd325df36c680a94e88d8217b06b0b340eca2e4666b3055532e2e7a7e8", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.261.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "06ddb318d690fe6c31360f318cd6d5d1665ef5a7dcf68c292db80d354585ebdd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.261.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "775905baa3f3dc19f8314323e4899f6362e5690432530372c09cef01952fb77e", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.261.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "76d6937cf1b4f2c7520c2d076cec8c031eba3bef383aab4210507ad0a7bc1281", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.261.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b11d4bb4192c6e34087012aaaf03c7d1e7b36d57d5602ce9ef538539cb0117c", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.27.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a0a3ec2145df0c70acc78030f17adc713e88176278816c31bba98dbdffe8c016", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.27.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5c36f7ea418a083ba0b56d1addd5cb1e444cfe5a3c5d51595e110498a8225395", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.27.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "72ee587930a428506d900a04f4b6e6726342fb499e8d6564b1fb52e1607853af", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.27.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "416f16a016dd41eed83c28537a0433e992fa977072a61426ed058423d665081e", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.27.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "87788905b657c6d5c2a0f944daa07245bc8bcbd73740a9d3374050eb3a8444de", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.27.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3e1906a16bb1a6c01e1a147dfa1f11595673ed8c02de84b28804a2625bee4e9d", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.271.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c1ac65b4239b61ab94629ffa69397270610868cdc684e91ac242f0a3e05e33a0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.271.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a5da32c3ec58294c26d5bcfcdf0006a312d85896b4d7759d3bac4c836f6d7a43", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.271.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "107ac4c16db3e954f386eece91883b32aacff83a32dfd75274a1e1c89e63fdc6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.271.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1512dfdd1dec2a2486dfa052756a4d25de6d21747330fabebeb545e67254c800", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.271.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "47b4ba4b96916bfa00b03d35bd544832ccfd392cad2c1824a2ea7d23ea132cd4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.271.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f7211e45b41fd183e38c0e73fce4a3d67ab4374825232809ac0e6dbe771dd2d5", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.281.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fdb8cda489908b2f9f82813ee2f814786a3ece2bcf043e6a4d4f6059239cfd38", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.281.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e03b3634bff3abc86326522ed8db10926bd2ddb1cb8f5bb5867b1458613221c4", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.281.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ecedb1d27a0046fb2f01c02786af5b8463efbaf893737643e0e0a1af5abc8681", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.281.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9909fc14ab50b38d351e1b5faa830baf8829c9c8bd506f521fd3e4d56dbbbbb2", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.281.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "74ed40890c09c20d622e3e4da075bed099a9ef173f14a64f3d00e76f7e922773", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.281.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "35f6766d5ba6a4ab2144c3ae11810919c285308552d4629b01efcdd95bf090b7", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.282.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b20bfa9fa6ea3f9fd7bc95027e7e480e642cbac22bf069405067470d0e3e17a5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.282.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d8da2c43ec7680f5f5812b35ef386e7e0d3cb54c9312d32c9a3be200b6d0766a", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.282.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fb71613f4714bff621e802a454f1ad6de7bc579e79d38423c2882d71dd6f1a54", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.282.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4d4befb82073a87156c8246731b45a5dd2cf6f9b0e591973dfba26fd1f59bcc4", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.282.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d34652965071ef8843411c7a218e102d2606075b7bda3d75a6388189f2652cc7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.282.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fdf7b09170ee667f8ebc496d5caca62d358b4fe43f274338ad3b60d9d0a3f177", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.5.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef86f975d38db9134d03158d67a05cbedc5dd64fcb26cace6493149a261ab830", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.5.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9826eb355e83774516de5446fd7358af99837890528828276d68f961e546670a", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.5.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f1c16891e5e72e9247b7f6eac66437ef3e9357741f2705f5ec43a04d141219ec", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.5.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7afa44e761ffc11bfa01674febe40c942df46d220dfb91411b63a385abbc7d41", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.5.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2564e0311e4a6fedf449858f5753102668c9cda8f6da5f67ecc4ea7482c24ae7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.5.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5262e33ce897ace42abacc0bd4c5930bb88cfb36d31c2a9d87058ed28cb62ad5", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.80.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c5f47b9211849912bc3cd2a8a013c9208a1be048713488c80986f6048c859315", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.80.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7fa06c25e9ee21c85f0744260cc51376f370dd82bc906ac23efd34fd9ed4d72c", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.80.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cdaaa7d301e2e71b36203ba145e91857a7afaa344df80f626a3f59e7e588f803", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.80.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fbb410ca3e5ea705024cfe36bfb5c32a73892d67a64350ae15196b089fe99e6d", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.80.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ddd89bd0689b7ecbb4d601d80206047681e1d6e7e15bd6453db4c2dec9f81afb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.80.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df89fe0c98a4f1e26fcdaa761e97d665d744e0e7393e85e6227fe767d661c5ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.81.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bd5620d74ddba4d020fa04aff7fa070a34f0bec136f36f2523d1e4020d43ca71", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.81.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e9795a631a04eb7c23d21259d5a9ed645e4b1242bbaf42956238de5615df1784", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.81.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "495d9269d5fa47b81db5c29fc72a35fdc6918d43b4a96a7f8d4de500611a9042", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.81.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b8a817105da0dc3794217d58a45d3d8a2ec609964fb141208f47e38e1bd0a3a1", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.81.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b1e1f21d66e091410a106381e20a21c981f49a7d760afdb1ab2b8da5e85956ec", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.81.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8a4847d291f26a4aef9c0b6fdcfd6be46199bdb6142aa8590fd16ca4342ffc22", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.85.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f1ec31011e80a999da490eac23db8489236fc587863d8fd3c8999f222e6c8cce", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.85.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2c1b549e9ff5925c8b9aae983c8f2346482f32bb38b4c9cd8f0de3c8f1365d8c", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.85.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18d1eaae5de10c6cd4306e7fc785baf183bcf79d5c455a12a5e64b2e85748b2f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.85.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "06709a1d4756362d99930326e7daaebd008de43944219ef5a82849c96150fa09", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.85.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5d8aea74c6e2b266921e00b56408d61403e4856f784c78f09be4eb7fca3f117a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.85.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6d0e4427d51066139b620847924d981b809391b8e14efc61add12538a5e5c4b5", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.92.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2b5534c48e395af404aa4de9d6731dab0b43b6de54d1e428eb7865e7075dcfb2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.92.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ce1299ac2a024ff264a3ae79c62b9a4d3e99bbcaab6e5eb2d1c6012350477652", + "shape": [ + 32, + 16 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.92.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "817169d361a33e8b067f887c857fc9bd0913d0fa4fdb36740224cab8710990fd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.92.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d167ff53b15b15c6077848f8f90edf6b0bae1485aee7f09ce029034555d1750", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.experts.92.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0e5f1984b8b182e83baf36a55581abae52bcccbaf84c092af3c7de13cbb59dda", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.19.mlp.experts.92.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e90e412fe19a68aef84f2131e962318567e544d442f42cc781caa95055d9ee6f", + "shape": [ + 16, + 32 + ], + "shard": "model-00016-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.19.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "23395d80e0b4a122c7485a447f8d9a03bc241be543a862a717af237cbaacfaa9", + "shape": [ + 288 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.19.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "82c4194d9d632a9e815616c21048a9f2580376a1066fb029123488f4f94f1724", + "shape": [ + 288, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.20.mlp.experts.110.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef37e0847265542fae958c3f4042523e958a699ca631ab702ca1f978cf131161", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.110.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d95183767c3fdaab3a8f7c2af4ba6c3ff6cdc547a0fcc131bb0c08e96b47d4f5", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.110.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "162961db5a3db431d77ad9da2ab2dce5391d387a56f424c3d47ae75e334e95c5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.110.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16e9576752530e536e779404521f732db6b77322b49180e803786a873c083cee", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.110.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "13813ec20e729ce8af0d356966691f0f981df4768881f7e050e445538da6ec8a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.110.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cde8678163307dabb3be24a5d06f35a709548e018b9b0fbb2290de26589774a7", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.141.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df183d44ab7b4ae47f4977d1ee1c4fd4690fa20dcfd3edd39bf45b251075575f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.141.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9c15208ab959aba81e7eb137523ab78db27b34852cd1f5606bbd3aafcc3965be", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.141.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "918a4a2ed8e00428391edc569ca5093c487dbc0bef4e825a673293b8f420020a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.141.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4e354e015b70c786bcfc3baf081402faff98794afce90d8a50124b188ecc9491", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.141.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "433d11144f5afdd393374a2260b60758796920fba7d52b3fe77a5211b3fd8de9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.141.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0a8ea443f762937c02829454eeeb0764e3e1873ddd3d8a541f170230ae7f48de", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.15.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a155724f1b6b16cdaa433392665a71384b7e08a67bb799a48aa7726d7cd5dd2b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.15.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "073195ce9e201112f1d18da2e08dbe45e77dac5f14c4357223f50e8dfc480e6a", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.15.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2f9e7790af604f83b6e2fb081a18a10d6b0718bf66c84f1929bc6429ffcb44e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.15.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1c0b358d4f06db3c3fbea746ba920cbd0def27b6344ec82ce309607bba42f021", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.15.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eaa0de4d95d41c243d4e21a98c520d04c353806afdc2c5646cb4dc60311fc412", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.15.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0bf0fb54ea2c8e21a209f55407eb436d7daeff92d3a75e35a02e9f25da9f4b25", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.173.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c784c2819a2efdea727992d18f82a6b2d181270f1b0a6782fe108857716eb5e8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.173.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "63966fe1d3dccd44465052e4a48935556b7752f3772326c22424b539504093f6", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.173.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "75bc63b184bf948196acf8b674b6d6ca545abd913b526f09589e828781018449", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.173.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9cd7fb13e40bfe6f4d58b4c88c9068b80da01ed694d39fe1d478675d360717e7", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.173.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4b7e437a70fcb91f8d9543b3616e176b3401f7806f3d608cfd4a309110850f57", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.173.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c33a966810369c6d9f4a6e7f14a9749da411100962a663e31799794163efa780", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.210.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ff83b489ba5cc7c3537df0ef9cc71c8acd9289a75795db2c80e568728a94d148", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.210.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd233c4b636daefa62f9103940411a222928f7de69dd6fbd4da156a8c531e31f", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.210.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b87b3cc4698fd4fb9956568f59ad6ccbdbe3ae8e9a180f8024e4dcbae4f13490", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.210.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1d63a5f89bb15eb0f2725f535b3d5b7bbf512081a62273f541d31dbc0b0ec3fb", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.210.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "82de9f876529017efecd63f33a6624f2be2777c42bd5b672246a762e44d43645", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.210.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f5b5deaa52017543d007d166a203d71c6f7cc70fd092da6517dabe87c604a23a", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.216.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "81b1e364b1716222e4d01bdd56d8e85d4578094a2396e3e86342d03c8d8d3f60", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.216.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c477a3ec13119f4590d2434b8635b4e4762c7943f633bfc1517b23eeff555ac2", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.216.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e674bc3436349f7ca9f1509648d7b42c98ba589359410c43341bdab3d900967", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.216.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e9b815a400b3dc33247145e86e70eb344fb2e588e8e87ab023aec08a69bc4d96", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.216.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2b9253592695bf9670b93eb80b6864c1670b05b9a99958b7e89fd9cb082d6ee", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.216.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "02a620d44d4e7ffb0a4cdd12faa9fe9a629b1ed2a2a34227cb3fc11a1c37a47c", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.245.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c254057372e57740a2d5817315063d87bfc8d8b369623d262b8c444fc8bdbf2f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.245.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8d8f706bd74a8a80c3a3ef35ccd41c3ed45db8ee53fa61f61c276294ae96f575", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.245.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b98de49c4fb0c68285f318d7ebf523331a19e98bf15c6a424bbd7c896141cfc2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.245.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "86da4c44645aea38a2bb04f96a46e5800cf78e6fc5bcd9dc1ef82814115dd79d", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.245.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6343cd9df75b7cb388de841b5608ff83b721b3598d24cb19de9c849311c86999", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.245.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ee5306beb97b6d58108d9602f8633bc53d6be6707d37fa4263dc38669974adcf", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.247.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32b3397be86f6e2e561bdbabd7dd27f446eb8ef31afb0544da26dcb04245eb04", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.247.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "823c5b3c84e19970a85ff827c1754853d567172d4dd469325bc6bab0d823c1f6", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.247.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4a7e1bc1926d89958f1827cad8fdfec4c35e3f6e72a62c896f795ce36c910ef5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.247.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "36caaefc148a2b3fbf469cd12216c84c19243c1ef8a304e9359badc6c1771a9f", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.247.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f83efb9a26ae63d4e6831e2e22ca35303498e125a1bfc4f3d84c7225e2dfef09", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.247.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d196dcfe2ce92bfaf1e2560194054e8aecf57d36267aa0a1992cfbe917a37fc9", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.260.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05b65641eac53075978c7859d61500010f6e3dbd510a3248e52d23cd0cf5e2c0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.260.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8821c59018d9596e2812d62465276deaa44dd0431637ed4baa7e46d02931a500", + "shape": [ + 32, + 16 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.260.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "603a7d1eb14f50a32209a099dcfa44f26683e9695b18310670656fc3a915136b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.260.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fefd3ee87bd1c0b40dffa65a29d0afecec35bb3db3bb3b23ed10ec3d6996bf1f", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.260.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "89db75fe533608ff5579adaa735cc1884c843b6ccb746e8ff20034e6d4abc081", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.260.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b321a599cb7b82a78e7ab621807249955121b2f3016759ee48ce66b22a6048fa", + "shape": [ + 16, + 32 + ], + "shard": "model-00017-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.269.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aee21326a6147b85f8476616df106cbd812bbd95f7985d07c2da53910bb02ac0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.269.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "788c2286397db6043c79b45565695f838784beeb0d92eda8755a3ed8deea4357", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.269.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e005742f40682f32105173469d43a2beaa0365a40695a8503cd8d4675e85cbca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.269.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b9b0924d2eb2625d13b0c9b51ba4ad488c4d058ea0e10465bd2c305288b4a4b", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.269.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bff298e01f41f346d4f15026651032e3d95c50512b83d3b1cdc5fa55c3bdf72b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.269.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fdd0a1f6be073eaefbb051275237ac352699eda690bb3f2786480643ea73f30a", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.282.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2d7c1e354ef56f4728358b231b66000f0264cd2fd40caf9a647cd9a287e54f32", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.282.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9275b607e4ff6a575f34e600a7eb33cd513b9463ebeb4e389a722ad0704f8806", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.282.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b9948449e917d1171fd2c8098a60dc39be698c65b1b98ecc1d6121f2f8fb61be", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.282.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c4e0ab6ff8973e009927107f3ab8c3e21525b5db0e8db8b0699d0a0bece74d8b", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.282.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "51ccf30c9f37b35a5fa9d92f72ac219f51eba0e8a187c054227863cb6efece94", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.282.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d6f5d62ca0184be0f4e6b829bcb0607d614e59602953ba79131e3488cbf302f7", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.29.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17ddf6767dfe82c910e8bff0a6b107e884b0410ea45ea2817c4fbb9c9597f94d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.29.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b0aedeb56b724deffa4fece9e29238bccd75fcaf150e1f62071641678bbebbf", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.29.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8f6b93d0a822036a224f7f101091311d0bec078cfa8738e7e4932a529cac2819", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.29.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "88137783ab7d1c0f02fc233583f7983184ef3e46ae2f18802db8b821337686d0", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.29.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ae350a1983722b7be25f3918d309fe2cbee59e31bc345cade7a3e003c9d31501", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.29.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "623d3b52b3d106c59f57142084fe96cacf3f867613d39a34e66d68340676c5cd", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.35.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b3e1b98ca1881bd0ef5a6cae0a5c2fab3b804ad97d6042d67351dfc3f373e26f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.35.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b497887911ee7b75399db2e7dd2749933908453660cecdc1267a7b7bd11e6d74", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.35.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32b5784bdc2ef330e139fb83fd6bc93e7553d8cfe6356114009f69dc55369c74", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.35.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a71d6a89a639170956bcefa27266c4e3059262a87b2fe709e17d94b8fbd9035c", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.35.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32a242023e6ca351cabe0c057cdd89f2d750b869095e6af4d528875eb7ffc366", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.35.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7804a216dc79761f6387468ab427e8d2f6aa8166a9a38c42465642d6ca056291", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.42.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0ecd2f801f80999d38a494b84d76c95c94b16f2bdd52a56cdcd4c30e369684a1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.42.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ec0f1178d6d0a4c99422dd967ba2043a3f239d3968c5036b8c3511ed356e1f2", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.42.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "804ccffdca194604b6c41bf47faa45e741712289fb50ac65bae07c1be30bb3c5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.42.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24068893a55d4cacc7c479c6bffa5d827a5d5a86a0d3ec1e5f38621c789ff7fe", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.42.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99a60c9811af6e0bbf8968f592b34c2df47d83b650f007f8c1bd197de14fffdb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.42.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ed5795c5151b05a8c95e6e9ba28f87d4b1f0c72394f285bd9e1ac7ccdd93bc64", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.64.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a48c7f23fbdc509d352fc3bd091b94706754b3cab45b80526324e099ceb3f5a9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.64.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "065fcb5bbbaa735102dffde8429b90582984f3a0846d8a2c3a4d2a0eaeba92dd", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.64.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa6c9ca9af637fb4e1ef4e7ea10341552c8090aa10be9a1c9479908b381aa108", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.64.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8faae817f327257c785831d6a402a5f32967b416e1cfbb4e67b168bac6352b7f", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.64.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "493650fd0d1d5c1089922037cb54cc2193e8c138ec6b363a5cb23408c70ba9de", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.64.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a49d2559a19822405b43eab09a8883c27f00e983c4686027d2fa4fba04313d2", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.92.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5c41cbcb9b2cc096863ab12e132088d01fac974e9ea9aae7c18881a80a555aa0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.92.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "35af1f6d4464a2d4a10072684f1b921459617bd5bb351f5e13307211879058d7", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.92.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3e0b8ac79c35782d006a0c70f05af5028c22aa92a525749375cff30f5aa0519d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.92.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0261fa69b6cbd20b177e46c447a1ba7b785736164f1cbc5c7b00bb9f37467ce5", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.experts.92.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2522b043c56863d8f387d6eb39d846ef1752cbebc8334c61e0011c321f057ba", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.20.mlp.experts.92.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "481f16957e086b6729d393fba9b326bda5ae5c61ddd737f509bcf4551bc82b4e", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.20.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "d1fcfc777f1aa156f2140b4d7b23759dbb0f7aa420096b3bd69a51d09b0e4881", + "shape": [ + 288 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.20.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "b7cf1b215320a88eefab9d7b417b886104c40076b712ad8de6063ad0937006c8", + "shape": [ + 288, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.21.mlp.experts.106.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c03d5a23404f8e2a6e0d9344f642a957e4273971131150ea7f77b9fb535dc2bd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.106.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3bcda474ee25e78a2029a465458047a8fbd947a3b8ab8527b3f3b4fe55a7e111", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.106.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eca1d5e456f44d58c7ca60c0ecceddddb995b3d54188cf2d6e8b8d172dd4006a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.106.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c103e29a615addbd8438bc31dce857aa26715fee83f8b8721b5f82bc85addec5", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.106.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "11dcaa7c6f031dd3d3b4c509a68d3e1b05dd447499ffa5f5b55e92d85964544a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.106.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cbd1f93dfe281288146900ce726362cc6e5e0839590d545605de11c6bcdc6cad", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.110.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "25e01efae44aeb0419ecb84b44ffc8acb3f1c54599f72cce5fd04dde5f0e9d61", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.110.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "21f2d3da275c18e4fa7733ce974280a02d41a4375c94c75358e6a1c924aab108", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.110.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8597fe996241630758d6ff96b7719c9e3de81ab1badb460ed900b25449f792fc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.110.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f774012c1d85bb2cb7923c0c6073d05f4f61346f44207cd9847d725248bfecd8", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.110.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f35ac4c756c1ec3d942aaf9daff899605a24ff866500b77508cf5cb01d187124", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.110.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df13059af2374f7baeea97d9e2f0dd85d3551d16a3b69013a42bc03f524e23d4", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.123.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ae06d7ecba709e6c17a41c1f05cfe113a1dd818c4b469dc8f50b4693baecdea", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.123.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3e58104f3f103f98410279e15f1a0efb9107163354da02d4a0dcfd1b95b3ea9c", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.123.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "efd43885694d4d06c2a18d8c883e8b14f87d953cc5d50e0770a4541f25573848", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.123.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "304df18a80ec2e614a3628b194c41be8dc6dbc504b1759ab57aac6dc15575927", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.123.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4ffa1d767e88ab61453cf885baaaa6dc868afd99add897e677ac30e9ac5ec6e6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.123.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5fe87fe8e3dfd59a7dc95a3a3b1362c5e8765102faa1055215ee4a465191a48e", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.158.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f6cc932b27ae0d529e6232a2bd797736408d802d43c8a179f078727453ff3777", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.158.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8acbed9930dc5f5138c8c26718b8dbb51fa33eeeb34b1028fedd89ef4d884578", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.158.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e676ee5e05e1cdf198df19e6976550a485c711d91b349b15d347668928433504", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.158.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8cb28d4c8f4429b5fc764672001cfa4c4069ac10e2e6f4ea71523203dea24fb8", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.158.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e9ce113a0f5ce5897ae52adde4ec338fe87b081d2e354b4d13ae023e86b00eb8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.158.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8df009e1833eaa5eacd2696a2c96291d0beafd8fa8b7d9e3121d48e4415e6728", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.159.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e335b66e2ff0667c3c8f893f0bfaf695b263d2172d7b85309b4e34f115237093", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.159.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3c87ed44f69aa199ffe7eb6ed4fcbf90ab0c98e8d039b383eec322eb1a58b28e", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.159.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ed6d54bbd848432844d0ef98edfa225124525f88fd0bd9841930b9f72258268f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.159.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4b07bfab605026f68136f462b2ae4712fee7c1f24da0a3d22dbb94b5d0b4f122", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.159.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fb5e56ce276588f0a10fb02a72fa661e3c874a25d29a4cb3ab76534a189d8622", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.159.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ef1e5076639e0c259b8e3e7c3a0eba4877fa8a1f846ab2b17112e9fc4ce9a7b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.165.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c9b753dcf2b9aa39ceb4225ca4b60fd5e1458efe31f6ef166e4aa8a23121e393", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.165.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0a6f1dec8f1eafd70b929d769c156d871d97afe185aff620a2ef11f8ed03c8ac", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.165.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "653d0d59d630e99860c93efdf9ddb825c1d58472174cf327be6e318e9c1c48ce", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.165.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "239bec5ca8c9623923d6ce0772869364c3c150fe3ec64c862f250981fcc73814", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.165.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c8c56c93ab31c2b780a111f90e6fc591a163902bf87dad9c4265f0298a1a06df", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.165.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2481503633ecc65ce834be67ced6fc1a5c579e378daa8ffc5ef4b687b7e27a12", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.166.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3c3923eb172d1ca8afa4285010113dc12ea3495d806a93fd5cd4e6782063cc6c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.166.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d62e284073cba1aea8bf86cdf75a6a4fb83f8711e54a6c98883031155ccdd6f3", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.166.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b55b63b9a5036507c3c8ab66078ace028a829c90761462b5d0b1fe4ddcc25480", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.166.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4a3a3b13a6b389bf9eccbe7d17e9dcf24aca8d487ead7d51066ee6509a3fd97b", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.166.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "90939df66782dacb219b541b59ad2b068f57d7220e35ae7537940418ef0cc7da", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.166.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01fe2e5d4de2a2bd80830eb53fe168b663f30fb25dea5df3a4eb5a83569741ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f3bb1f90971d1bdd7c201792c94a459a7e02499f412024078feaa3f4e333d97d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e11363913be61d311c97621c0db92783baa81af6890e836a404cc5c6d0ba7e10", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "662eb0b040b1ce9e4fbb52be40029de554ff2633b931b4740a17fcbda96afd16", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dc4fbb4554a266b3db95cfbc8ef3350eb05b98ee6f3572f8d3021d7bd1e0a5fb", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce87b36cae1b72b0778fe2d50b625e67f0ac51a933638e283022dfac21ed408b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c0f144a8258f0c1ff404e022ff0dfcb920ad0f5ed881749f0c285b34b4afbc9e", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.182.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "11c3e8778aa520fa1ec47b2f2ea22d611ba5b1d79f3b86a11291796010a2b714", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.182.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b1c3fb61c2ac1fb46119b317768c0e0ca1b651b4349b7de43cc2e4986f2d53dc", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.182.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4d3ac557f850280a9cc4a4d793e8957114266e8bae5f5bc1755236b9f0497cca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.182.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2381b06541c7750ff801f61865e05a01d7cb747389dabe39846866f2ab9e735d", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.182.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "66488633a20db88654720f25f65af4aa7763626ab133801cbb4954261745aa42", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.182.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "09eb68dd18171c12eaa08636c72d0ef66a19981d52b2f03ecd8573132f69e52c", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.186.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "75ed6fa22c6995e3512e6e0f88a94bf64f5e049803f1b2ec9c693259c6d137b8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.186.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "895629900ef5c97198e51148b152e393ad2650c9af946c876940dea7f74fd97f", + "shape": [ + 32, + 16 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.186.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c12cf07489e23928db092dfefcee1bbcd5e9968bcd025ec9f3186f1f5e1b4018", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.186.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4e708b38961fba28f557bb6366315b3ce51c744336c85a0a640d63549e87e720", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.186.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "601b06171f0ba36051ce8cb7d198f4fcdefffd748ff0c7af93bdac168772795a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.186.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4da912c20a2735f62090e0e1dcfbf562df86b5187a95c37690462cac1ae0d61c", + "shape": [ + 16, + 32 + ], + "shard": "model-00018-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.211.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99232c4fca60fdd97effcda4aa0641d62252b3107ea392b92f086be94a18964b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.211.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dc23cde9adfe06c687df0d04d24c9f6cb33082b48634061fd7570fcd2e319c87", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.211.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1facbeee782857bfedf521498294513b4bd2d6a9c439ae0d0b536fb7e4baa8bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.211.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4fb7767a4ba7b364ba2791eb6a1b052bbc94cf784fa36406f971856fb9b314ae", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.211.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e49a27e1051e64a2bc53ba3e5a404544e43d0dcf0606b70da52c1d945c6ec0c4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.211.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c4f58b771bf2e71982ef8f81a1c9ace54afe4d8ba629fb597e2a62f3909bbbbf", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.267.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5aa445931c2d2cdafc2adf3d1a33cb5dcf0123c227245f469992d6cb718c217c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.267.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6a32a6eb54c0b7c059389aa0d0ab88edf17ddc509082375f2558230226d7be67", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.267.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe974b1fecdec5a09dd54db5a5a62a32c3ad6801eab7a771d99fc3b91f77b1cd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.267.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "655493a6f5e6902f821b76fb42a63fda04b34a102b05284359accd81050746db", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.267.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "43d56eb8b64b75bc2a9510ea94241bcebe1ae242ff3511a20619ffb989497b4e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.267.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c1980546d1d2a45157d9e92147792334c25bbf3247d0285674b5cafa31c420cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.33.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cea18d338679a400cb6be2605d5078e56a3c4ff0d9cdc82dc0ca75394858a2d5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.33.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0208577bbf36206e80a810106c4f6e0414aeed0200d385db03b2654ed1c9e4d", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.33.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a53591c651577fd1f687424190601cc9b982849bd6e015c9ed8df69a88db6dba", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.33.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e14b5d987819fa5e2e7443f62903192a03a7dd12f1872b395c2f173531ebef98", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.33.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0170af0c6185c9993e9139ae03bf13b06f5c46392fd4170a00e445d200f07030", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.33.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "941b78bf297543aa1336ba53fe9ee6a6b7dc552d640f9f9f3bc09c05bdc3cadf", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.37.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "742c754ab2c6c8220104e7c66512f59a737ae72a393048679b2b2bb3aef4943e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.37.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "53b6d5ef132456c29de8cc60b3424fc7d05a2fe1f282069e1498f91a67d859a4", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.37.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fbcd814cc4371416a5f385f58446f7508a146e94c2d3c6f5c638d3ad0c27a138", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.37.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f17721ff29fad6aef0f6e95c7f86cf61d5676f9708cf1965dc658a165aa08c53", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.37.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "35ba03d1d97c4fc0e9f5e835b3dfdc80871adf7959449507946a1b4210259b6a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.37.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "66518dfed0829f52cb7922280caf2d88cbe1c64cebeedce5db7b52fafff71682", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.5.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "393267635567177d44b724cd38dbcc6045db8fd6a993e942e50db1d7f965b7c5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.5.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6a49b1cff91fd6d94169137ee7490a047149c701dca76b2a0018b9dc529deb33", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.5.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77f2f9806d754bd2e3c1b8656fd86dc30b161901cacfd3e3b0dd7fe6fd1371a0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.5.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "014590eb3b43c2c37951839db14b8ecee3a8cfaf60c606d644fcb48270a17d1d", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.5.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a7faf100df96d1f320909e5b9894e35d1088535a255dd419ae02cf64e9d8dce7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.5.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6c1b8a2c4c48c8b673742b917d4cce167d3ac3f04f5ae16f9f7d86f1ba88b3a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.63.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef479ebb01fa28bd2106cf4ea846c7b8e458ed30f0ba7c8b6286942660123b8b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.63.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "911a0d0ca8c276a5be4d802a4c32fe8e8cab337f6e5627b7c87f908a0c12deed", + "shape": [ + 32, + 16 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.63.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f024a645209dbad73959c87872de3459ccf7b8110fda93da1cefb4c3ea73410a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.63.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "447bb54bff670adf5bdbb58b16143b9dde1d44d8a8b9efe3abdb06375888b58f", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.experts.63.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "98f77f4bc068eb8b678cc6b09fb5e2dde5f019e4be9195e9dbd51fbd673a4e71", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.21.mlp.experts.63.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8f70b3a15942d9cc99d9175376b3b1590fdf9a6201771c76887f9371e8ab0851", + "shape": [ + 16, + 32 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.21.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "c4a701165bcc048d598194f3c5efa0c40decbbd0aae8d5b91163e19866b5ac6d", + "shape": [ + 288 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.21.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "eb01292e2a918b59059a702ce34700ca3b4e27369f49266b51549ace5a72ea99", + "shape": [ + 288, + 4096 + ], + "shard": "model-00019-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.23.mlp.experts.10.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31a8b9675f6136d3c7a512af99f7a640be1a8f700be4d70882b64e2ad7981eba", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.10.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "95bc470366d51c076cb822f31b495ad45b578ce8403e2eb9fba6e7dea02eca15", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.10.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bc1bfff92dbfd829315669baba2540727e811ef08a3b3cbb05a6d727e13fecb4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.10.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de9ee0fafc3613427da1e5754023ba341069a35a7345905a665b6e18fc238e4e", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.10.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c6ebf9c811674b63d443cee637c96bf453040b5da6699d32480c5c74f7160a42", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.10.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9cc77cf6217c2ea097a9fc34236d20ae41c42496db758303c3c0ee9249bcba03", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.106.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f7dad595b45e6d2adfd154290c0157d2014c887cbae8b31ba3df1978bc291f60", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.106.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4a6b2a40d4fc6069da36dbd671d095849ad802a99b2f0c8de38858e5b82cf095", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.106.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f811fa8f4d674991c7a7a9bff788ea49a1ad71552bec629c6e5d34c766bac628", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.106.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1cce16c69c38044b67faae628bdfcc7f8f747936ea48a7080994c901c466b2de", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.106.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d9e3e45ebec347d4e124d35ac69d03d668b527d02f0f7926e0ccb6a76406b2e4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.106.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "224cd4b6410a43d2860f32db1e9d3ca4af2a67b3e4cab89889927beeb7176617", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.11.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ac6152f3b4b37f0600e9e3b8a0ec39c387dbaac11426b58a11f14a2a52057e7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.11.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aebe9c16ef8ce85dbf29e922c801500ea964ddc5c7e13d07a0be70db5c031728", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.11.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6683d520e291d4276ad92af00e25747a2abffb0ed9eff546b1477db0bfd59e1f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.11.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae2b1cd6603f9c6e04eb02f7731cf5a0e04630f7cda8cb66babce276c7c0d2a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.11.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "55bddd5b76a48f8ed944e47b77f62e5540e143b2ecff74097a6b5ea85b499635", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.11.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3f919140a5909c96300469542f6866eb07ce2a124552303d18e7cd2fe729e448", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.13.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f6dc86eb3e057c13b3e4654577cd296f66cad9868f53deba01a7caef51e8fa88", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.13.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d1fee05fc31f0f8d2c4bac0a645415e67690571095a0a00f0060b3bf0c13eedb", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.13.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b1775f109f058e32d7627cc579b2f422a6520069e9686ad2be32dcc554b2be30", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.13.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a71dc42dc623f2bcd663ebee7b91490f8bfed9b556e0228a5fa2f77a3538bab3", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.13.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e85e8e2be2695a4494aa1bb51ff8cc838e1a282f36cb4b96492ee6289ef7605d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.13.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "904f06ac34cfa51184c465a09558cceb4d27771ed5d81612cf56f0ca79b002bc", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.133.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f72baf4150fc0e7e33d3d1fa3520e6606f12ade26214be1b526a759059cd5bf1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.133.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0399006ed33dd62cc7089709c53742469e1560d11a37d6953676d0c3ae7f4768", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.133.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c85d876440f95d33019422fc9055d327dab8360e19bec3e1e515044be41791c8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.133.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "576fd9f85f996ce6a8cf2b97a3aace24b9cfbb99a77ecdc67f28600b0f5f8d81", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.133.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3964cc4c082a85f87f43c3b96dadf7cd77a334f34736b819eb4440a00b580780", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.133.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bf79fd29a65740a0faf4b69d141a0b442496748af214bce4cccdfae49d5f5457", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.152.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "382bd69996534705322cdb57e7a3dd923bc4d99ebc2555ac10a2d96d68b1ed2a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.152.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "128df7c3f32004582929241492749ca7f4de6facffebbc796b8bd7025bde51a0", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.152.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c1aea854f7b2b1cf7435e403e0685689ed635e9d35a3b7e07e4a59d42cde7795", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.152.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ad2f8210a910b2e0d36f17891326c82e1d5a8891cb37e40a2a01f68cf5e4da7", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.152.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "07093a2ad0ec81d4838d3a38d491c534ff5fda15df4afc96dd352820b5094e26", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.152.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4fa60a6657beca65d4646cd614e1c987f1c34bf88642ce8a2d71cb18447cd3af", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.202.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1bbea70869dc989fbecaa8ec8f9461f7891e30ab2918080dfcb1c1d397cb4adb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.202.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ceeb33212fb0af6338ebe5bc1daebc03f372a6b6fac0d84046e702581048f57", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.202.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e84197766e31a56b78ca62cbcb9fce7c22c21fed40f7a9d383f7b7f78323dc6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.202.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3c6a3c4bb608744385c2b876cae4cbf2cc88186a4c90bdc33eac6730bb4be4a2", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.202.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e05a5d6ed2236f1bf407dee4c7038dd104b2fa9807e96a4ac0f0413ffe0a138d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.202.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2e255c5d92474a2054451aae269fc29eb12da16d3ab7d0dd523a540024e4119b", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.219.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c9489061b24db6c39f3f7c81d3f2426f00a48de0d9456180331f63575125d90a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.219.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b2e07c16603e5b16d294305d1f87a6c7e775061013d0a404c22a781d49da8186", + "shape": [ + 32, + 16 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.219.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e986f2ad7b5d6c990ca786ef44232288f526de6efd56566a9bb6aa5db630cac8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.219.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9b85b38e7bf27854f779b82e69d8ddd7f1744656b9934a9231550447d82e40e2", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.219.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31ac78097ffad46753cdebea30e734353e36966f01cab4e98ebf47c3cd142558", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.219.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b2b5729574c162e4ad14b13ae1bc16bf53e3b397af8b8dd6750a27f48e5e90b0", + "shape": [ + 16, + 32 + ], + "shard": "model-00021-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.229.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "02cea2c2954e3d9115dddc7b0ea00861210f7bafae2c4e68e6f2cbd8f09bec28", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.229.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e8c57d40d4effaf09bb695a21c36d6d1d6d28d0348c16643df040c182ae44218", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.229.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "590f0a4ba975a021b087aa0730066a022a73396161958a787001e6c3f3b36091", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.229.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "64ab662756360c4fc0a400e60247135485ad8a7ae92e72717a425f6787bd3853", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.229.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "444dfaeb226161e2344148ea60c69819a25a20fc095dc9f9ef317ffa372bf5bd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.229.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4c091b1e09ac1e692ec534d83758834a5fdeee2397963516c01b41a576cff746", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.234.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ad5983d69319b21c7e4be84b16ffec3dcd4927a0d1a4ae3cb8746af310d45dc2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.234.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "85b52d39701e2062fcd29bf6e2593bd3a1a7902e6ffb162e78bb5df6243486a5", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.234.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e479a2375f6199b5724150431eb1d71a2f4db3b750cf538339fed6605e808c01", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.234.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "92bfaa1b5f555996381cf9edf9722e6a749be0b3ad2c21aa00d7fce4eb06768e", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.234.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "15ed70189d51279ea641ddb36024ac19106c0fa39f08208a2b1a5f35a08a8de0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.234.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24e11cc5a8eb0a906f896c864a4660b834058d399cf9955a4495f800ba683587", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.254.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "533463d3c72c06733122e7b244f95f48f89ff74e3bff250f45049731197fab71", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.254.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a24e8ce50ae29b1f1cd16ef6d7f1c77b9f4d8462d25d8eb06650b17181675fb4", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.254.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f9f9f924bd7cc7b669fc70aa4ea1db365a719ace1465ca2fa6d90a366c3bf68f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.254.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f272ef43f29270b7d74dd98edb7db65271a856a53648231e9a502360391e1bd4", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.254.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cfe35ba978fbdeea949f897abb86d4ad9e9e32f5fad72a5a7c0cc198c04269fb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.254.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6534e451d7d51ac734e65bb03d731248b2e79b0b7210041a7c20499d4ee29aac", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.269.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "73407ae5970e04ac162a7624b4832e69402f2feaf8ac83ccc8a7db246132a323", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.269.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e57f16f688b8fb98e76965a7a4d2f2d99a5e6fcf938408233f8fa7962e4081b0", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.269.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7a461065f55b6e8e4e6f7814fa7d9c4f390928bbb8c2c7c9f7b9bbfa13069b35", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.269.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1a578313d8468f32ddb692daccb4fff46f582c6029fc1084423d36975c26038a", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.269.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "48dd48f973068b3e10152b3bf8649a87971b5fdb662276dd03d6896aec1209f4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.269.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "173ed8fc0109f14c7f640bbabb6fb2f91d4e1fac0e0db167fcb2c9b681abe7a2", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.57.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "10ca67edf7a8564a0b6e4957731f53a2e5c3b3504007461df4d47b745c13ad85", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.57.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a3d15bc0eb1f29aab5b8342d0cb80f2cef71016650b67f2e37da6166bc5f927f", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.57.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "73fb3c18dae89e80b15ac77057792cb22dff2086bfe1cf1ad47782167706753e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.57.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "acd510cffc5165791fa1004cf6f50373f97753fd1823f321962dba132b49ab48", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.57.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ad2da1e5c947c7231973714047965977a95f2848ff8dfa55bba35155b30291f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.57.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f441c3265fcf4cb47da96eb0f4ac5e107f125ac85ec694f59aae3c1011ed3f3e", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.67.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18defded257b1906c2df3136405e686ba0e5de3628784b7c1b64fdfc3b835bec", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.67.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ad1b0ea2df5103b815281f3d7164796cac8148f6b448ad5985404c4645ad1033", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.67.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9a679928ede319d68cced0bca609266b90b856156362bdc0055b3ec5f944b957", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.67.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0c42bb0bc1b9f158b125add674a0ce54193fc6b4555e7a581b4af88afdb125f", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.67.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3e73120def14fc30b2ffaca3c5637bcf310e0fe5668105ef7170f82c6d208936", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.67.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9aa1c96c0bff458ec5309cbd4900deb0adae7e12ad24e4e8922980fe4bc4a3a4", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.70.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "91440372bede605a8006fec09c13ecb1acafab7c7cb3b8ff953e4ca04fedc7fd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.70.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "11fd77fe3732e22abdae3b4f3c9990f26037e97978cdad8119d957a71fdde878", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.70.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c6f47ac6f3257318ccc4236521baf07cc663542851a7376f2ee3328f1457d3c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.70.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0213eb67b21bf251899383dfe1e3d970037be7fea9d2f0f984df15d597d33a66", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.70.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9a8151f5cb6a03d7ccece878de025d7312ebb466e3f4438d6a79092e87c574ae", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.70.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "992fa86c56f5a8da1ea8bd87d94a6c2968d69e5453f1d5f09414c7a68b0ba83a", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.90.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe636c801b6b3a36171d630d0377bc4e8477744728d684c1fe64dc585b798e88", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.90.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "87b3f92d2904c1698f04af0a8f69fa04c7db3652d1d029a0cd931bf09d7473de", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.90.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef0eca89df82fac9a20a9cad2ca6a420abf45a4a41d2b99db23bd7ae3388a47e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.90.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "75914feb871e54fe4af735da0f60d8948eace143eb3c3750fd2b4ee4aa2b148f", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.experts.90.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05043d58bad3bf2d6933da69563987dcfffe4bd857d7617a71c5b464fc2bd3b2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.23.mlp.experts.90.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c4732f4bad118e69e0fce16f83307b086d6d2340fb8c6d819017ac054bd623fd", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.23.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "e6d317681d10b281560987247f4803629f238189aea1b09f4b31afbe97ec9f90", + "shape": [ + 288 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.23.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "0a08a097bfa59446501291420e98e823fd545af6552c4f1c2146183d74ca61ef", + "shape": [ + 288, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.24.mlp.experts.101.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c950e3daf700380b20cfabf2a1d03ab43f9c96c893e40595b625713fa5231a8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.101.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f9a91b5b7ceaca05d9c5946466b3fb000d3a70c93ee604c7f796fa7ffbd3bafc", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.101.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e91e18dfd95ce26c6d4ab1d0b8cd57d8e558b28f0468c597e782803c33131d45", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.101.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ad761ad3f6ebbd668be34420ca22a9b7f5e44d78b74ccd8a80466b23b389709", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.101.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "37f14076d3b4a28fef1f0b31106ee4a30c1d6734e7401b5ff12c1f99522afd1f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.101.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "23047cd985663696a6e828f0753569896c9e75373c3d486d24ab8f83e14c805a", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.119.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c478dc354ddd64c615e528622c3b9c75d3531653d7436377f1cce9eff64b78ab", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.119.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "05b5f1a9ee17d70b267da4a4c63e6376782c691bb856a561b666b0e1d75e6522", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.119.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9e4d047af938162a5ead7ff39b7d0c9d002c7e04ba84a2743abebb54502b8ddb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.119.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cbfb8588586c1f2cecd44bc28325f6c9e4969f99cae7672269e2fd0a803b3855", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.119.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "409b1e6721ddb80c4c9f720c6b0ba3a80df30c56f0af43356e32b5c945a4f26b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.119.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a224c31744ce407c7562ecafb1115b40ad7b6ea866f7cd8fa4f16ac58298fd7d", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.135.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dffc628f69815e35a53b5a47d776cdeefd53f2ab610252a7cc4f51793a73ae32", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.135.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1a263dd626d527774300b1492b196bc4d23a3b4e7272a0932ca00c98a155ce0c", + "shape": [ + 32, + 16 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.135.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "641725a86266fdd90cae85d44bc38efff7b286ffdc42f01bf320a7af096a8062", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.135.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24a52c7a3d9e1ec3e218668eacf1e5bbea4f3a9b0a5c010537ec375bd1ed5f9e", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.135.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c6febc93fe27df9fc798a11de727d98fa60d71bb3687f44a0d83cfefe6f4e48", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.135.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0ecedb47220b319aa878a2665b3dde9324b06351612651bd805a66faa4dd1bf1", + "shape": [ + 16, + 32 + ], + "shard": "model-00022-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.159.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "572a1689f08bbbf0db29ecc432255e57c6069983fddaf071bf3435e15fd5ac67", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.159.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dfff6498a44f59bed7c6a4c407d825564eeaca041b6214f78157c4c0944ba4e3", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.159.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "95f5d9f67da9da6bc177bd706dc8d95f1fcd4db0beae66cf1d8de66086a125f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.159.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d7b490ae508b455bbb0ec595bef4c8bbe4023fe88e37784d880465ae9655718c", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.159.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f104d7191564dabff0b0b6fa67f61e67831eb61d09dbd16c18aa45856ce4c960", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.159.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "95cc3b84ccacf44bfe11e57cbab65d8e08f7d76805f13e6732ef3728f72622b9", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "757550cf514cdf51b5d3f2148d22c3d8ccee2c3ded60161ce12ad7c2fc31236d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae9791efbe581f8f1914488bbcdfaf857f6fbf92390bff98b8519850fa4b8341", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77d869f9f7ace7cee13cb435558ac33abe8e505ec654f5328fb274eebc2a4a42", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eb9237bd8254b275da652f0bd996c9d326a78195054965149324f903bfb24004", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5fc131ef5e22281b0177efc327e6df0e524f50397fe800d6bf51847eac06eca0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d66ef9f583787dfffbd4e6c897d6b8ed8dc9320eb7e7ba0af5035686c386e90", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.193.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a505386df041df5e6e856229cc0d35962343a4f2f01a4884e4798beeb4a1b700", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.193.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a90605028d94e36da453db4dad3153def25f53bd0693b86dfc92b6815e899f60", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.193.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bca2d64502c21c1fbe405fa30001afc28c2e08bccda6ce9370f6c09d132d7523", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.193.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ab95753ab2f4895236dc199f64f4f785365e556815a61eff74288fd32976f64a", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.193.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ff84cf8a0d05bd326943b5bbfb24adc3e1168495ac567e0576b17942a537f6b4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.193.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c9a89ae629d3ba812ff949badc831e8632f4120d504584bfba87a7bed2f9c5de", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.198.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc84b287a6f893fd1829b04796c6c054fd5d3dc2ab4988ddb46ea846c7814ad1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.198.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "abc7d2264084b909aa455bf11ea47f793eff70832ae77b791e09d4c492c128f1", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.198.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "51a2c0c68326acbb3f79166813f764fef4e863e050545930097ce08b16d46bdb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.198.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a667201378e0684b57954eb269ce1be32d33f268975bfbde34c2b8f332132cf8", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.198.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b3b35ea03a554ef376b8967fa931a4df558f12a4d8ff34a9754bfa0716419c32", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.198.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "922cffa41998006c69d2c950661d286dd95f50010987f4cacdef65241a24fed5", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.207.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "00c5e05500ac0b791595ce4810c8298286bd6e06919c5ceed7031e95303c9ae3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.207.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "476e6f94ea5598f071d8b5918cd07a42d614d4d3e6fe72619dd4408d147dd543", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.207.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fb2079b17e6706ae756f11bfb2adcd4cf2e7785756af6ece6fcad31599e1e19d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.207.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "18cfe45298ea67317feaad6859f3c1acdc511a28161955c28b92245791ce7845", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.207.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "63efe9cae02038ba20cb6a4e1bf5f7f83615e2ff929e70860b30f3dc827a2f71", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.207.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bba6a6e42a37a7b4f371e2e58d3785d01fa9ade5e603e4d1fdcd2288074fb771", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.23.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "35c2b1a9c27fbddc1199b17d731e32ac1a2d0fe8f477d8a8d060cc42cf28ae94", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.23.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2d30a3b1d405b7d179c14d5f64859a7200b038f5027961d852df3a97fe306217", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.23.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "08e9ea4dc753957b13bc1e1549f1729de15e7401105a0554241c0e5e256b526b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.23.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd26cc4bc133cb4c44bc7fb8afe27f1903fbf578fa8f8b1d9bb0630482355eca", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.23.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4aa6fe1f3bcf9a27f0be8552ebfed900a2ed682637af3d0e575a7c3d38955d52", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.23.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "59669e58df77e7c0095902cdf3a5173064d5b61029ad87ff21cfd08fcde443c7", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.25.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "379e52425b5ef18eb6a85dd5848a472decfe00946c6d82da8e788960b123f2a7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.25.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e67815728be4f6a5e3c1046403563a4fc925defac249665f535348d7cc8a8e70", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.25.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "edc9ad43ce08682ed0c3d19478f950729b5b667580cd88778063a320ea9f76bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.25.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cd0200b31e2b8da6e0f34ebbb8d2feb894e1b2b3ec4e2e1866824661cb3baf1e", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.25.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9a2738f040d6e6104063a4ddc277d4bfc40feb7f9d07c71a4f9f4320433cd001", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.25.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "39a1c26f98b592dbce8a87ce2230c88da0a21f1ebdae378cded7103d02936e29", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.272.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "24859a26a57e7d95b8984521ed468bf34ed12630626e6705527d81c95469d7e4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.272.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4d112289a33cafb892e3bcf6b1bdc19687ba55e9d2cfd21e0da52edc9cc66dac", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.272.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "21e81530643b637616384d926b258753fb793748ca894faaf9ba25c82404c6f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.272.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1392eeffd0cd9d4ba4df8fd50e4526c03dcfcd90b3f5602c40702929ad332100", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.272.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d09eba04aed30bfe2dadbb9093844ff506ea77f20a554e97300fa2d82d0615b3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.272.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0d23255668e2d0ded3b7b0f0e3425074c3da6bf119b1a7cc3f68425530a8a6a9", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.29.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "03e203eac9dcb2919a0381d92874e6e96ed744199eb973e3c04cf42e2f103319", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.29.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "266431f0ad887e79c6da55bfe29346612a1de28667019864912cec4be7b11311", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.29.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e88f4bbfd73ac02ededc60a1719647e416ebcf7f9462be10cb5d2304c14c386", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.29.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0259f0fb3a5b4e966dcafd508993eaaf595baf27b3ba7bf33a6d433e96e19dcc", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.29.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b12bb298b8e4ab28a4f11caa806dd701165f62171060a18d81583359e1fa2732", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.29.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "038072e1cb512dbe4b019d31746189b13cbd41ba22c70b7e57a4d94f43e64c8a", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.35.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6528c2338fc4cf2e26ef0db415f2bbeec7d9b0c2d73bcba3d1d79ff43df93969", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.35.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c5888f492f09db32afd3caa0a3a47b601c9a9f1c410bfeeb07554dc0f7fbf452", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.35.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a01684d8912c5a83837ef96e830d96591338842440dfa8b5df7f0461d380d35b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.35.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "75bc59ecaf77a9a620539011e9b6ac77f4a8bdbea8f64da341404be5432f3748", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.35.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e67784bfff2f6ed0fd0b5ec7e3df4554a802d30f57a07d63f41aa5f9e8d6e407", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.35.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "093b63962ec8052db4a2e4c98eac0ac8f5358d1e5e2c9420e36593edd80b4792", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.45.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f85212938ee72dba7871842f1bd53945664bad22a94d13ca8091b181c7bdb922", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.45.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4b3ab3775f9c4ea27f524830398d24c069f0a32a16bfc4b25132661c2afa4247", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.45.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e1b1a625d8a7cffc62bb3d53dcbab3570f75af42b03aebe0e66c063eba05ba9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.45.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "180cf1cae9774ffcacdbe8cbb1351e09bbbe7e8e2072691857ffb1978e6d64d2", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.45.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bda24cbae7debefae0f6d90656a66a85a93bb01761d46a7f0bc47ab230c34fa3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.45.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4db05daa8617cf616958e6b9fb290b94905856165ea66d13dd3395a5c0560df7", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.49.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9527ad996cc15f01bbf6ccddb3763c2fbdfc70296d563a0c3540c139e1b96472", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.49.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3fbcb97005ffb267e5e0eb4e0a321b8e62b37bf0b8d6fe533facc6bf7d3c6c16", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.49.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce0eadadc3d24c3a4dc0ab48ff119f08864189ef16597219cadb5f45c552705b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.49.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8bb04f35042a83602798fd3cbd617c87d33ec54c43550e1ab2b2c0165616b231", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.49.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "869704d2735164c6e81350cf084172af15ba4709176816af6d1ae7a3352182c0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.49.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cd8c9badf403ebc2dd49569e1a81b1e6cee2a4062cbe8a80e6ae3aa460812b44", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.56.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "12d1950dfae1a19416467a75914d4be91562ba65d6af72ea1913f8cb69b0d806", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.56.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c57b075e5b62a65d099d129c6f09c86611067d628018039d850bdb9edc92b180", + "shape": [ + 32, + 16 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.56.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f4ff0e9e051e65bde7be43974c99880b0332f46bf81784a0405aa08b1a017671", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.56.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fed33161fd375bf72428ad7351fa5d738410c8ff459f6455dccc79d8ac50e8d2", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.experts.56.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "62bd717ca56ba9069c77fadb95da39dae8c60082d5f7c2950d8bcbfdcf163fe9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.24.mlp.experts.56.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cc35cb5f0eeb688f95a2471db221ec82850146a3e27d6a756164803ebadca8b0", + "shape": [ + 16, + 32 + ], + "shard": "model-00023-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.24.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "103bd210a3939b939aad95f63131608aafe266f60214ecaf62be6c713c8af9a8", + "shape": [ + 288 + ], + "shard": "model-00024-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.24.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "d3a3674e29c0855b7dde119b9b90a371db8f922c11d2c8c40f9e012d0439f741", + "shape": [ + 288, + 4096 + ], + "shard": "model-00024-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.26.mlp.experts.127.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bb3557e09f55b20e243589ddee4cc9df434c742c25f930a67920f13dab795793", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.127.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0fe9ddd36488736535b5be53777bfb2edbeda0c2c1ef7acbe36433ac9822b8a4", + "shape": [ + 32, + 16 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.127.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84523ff3797322e92133a92221a25c0aaf991ea89cf2956b60f838a986be44b1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.127.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ed1352c9d31672a126054152ea4b54626456aa6b35c05e5bd5de37925dc30176", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.127.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "771f3e87d765358e600db9ef3794758092f17f3eebf1c989d4020b3d030d3eb8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.127.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "434754a04708b548df837e3b2735c3dfa2cb60b0170334dfc71d9949a222d612", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.145.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bc62d8dec3cc8dcff0895e9faa3b0b5f710a83549c792856227f51c1a145b97a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.145.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0e666e73fc33df5f9fe365fe6545e10202e8b44d022dfaa74ea32e93a23cff6e", + "shape": [ + 32, + 16 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.145.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "24c65dc74175af03e43f6e89a783e62491b91f362772c8e8b86a6be8ef33d7f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.145.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "43714d53f141b8987e9744b3dfaebaf5b10654b0e60dfb8c97419b1f872d9152", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.145.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6b7b0a183c90978d9157985bdb93e9f16257a56c3f9408cbf24b5900a5c7ac18", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.145.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b94d99a6eff11b35eb41000ebc4e81b73dc5eb7e87fb921405b18a265b8ab7ef", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.150.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "69154b1f3f90ea170c787fec062712e24f17df0830522a3ae2eafcec26954e6a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.150.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "add6066a123ab01093d048fc0b8fb12b2885b112fd06e375df28f5280e15acc0", + "shape": [ + 32, + 16 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.150.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "af65017f33396a9e332a89a8537b376b351953c1c4b3a709c2258f17cecc7180", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.150.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4eb6f44881f21e280a793ef1810d24cb2be99f59cc91a20ddc99122438dc077e", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.150.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0ab94dc04e50e5681b870061d0fde8c907fe0ff491ec5cc92a379c887d037a2b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.150.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "80aa058421c87817ebe4bf987db74229b2b617a96ab2b8ffbee21f0057ad0590", + "shape": [ + 16, + 32 + ], + "shard": "model-00025-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.203.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17ef81e45d6531fd3fdaf32012f03df466d40b30e0f6e992947eda6758483346", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.203.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "95d82dd69785f6889babf814dbdf16f1b6f579474dde9a9346ec518ca9a0038d", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.203.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7111c2bbafebf13b0adba8a5ce483e37e018dd7ff5e082a7a4c1baf426efa167", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.203.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb1cdf2f4cad5040fac42519bd064250538e6d92fd3b514d38a27050b1047193", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.203.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "81330c2207479feec4c9626b898c2a1a0ff4aa8fa1b4d500f7435d2d3468814a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.203.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b0f545f2c360d391fe66b469a9875564c631bd4213b195f6c67d41bf70e64cb", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.208.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a639ddca2f35133ac75d72fdaf2959701407167b637da04c719ee8f5a0f07ff6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.208.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "66f5d148f311d33e736b354b7f4a1710bdee7f47934bf20b912be3601ef92899", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.208.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d07abb43f7faa7575c6cef1114b351fcb40756c52d7f15d2504c20bae4606ff2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.208.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7b0ddd5b6aa989828b29b7b5bd494e4c0d09dd74a9361b244bd64383e64c6ef5", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.208.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eb73726dcb60223b2950c80e0d1fb4112b92dbe9d10b97db47c53a6dea6d62ce", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.208.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fad098eef3acda6fa59cae165a6d96b3659b0e8d5ec047081a8f31f7d6aebef9", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.214.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6b89ed73a83c5137b02c5dc5fa5f765b0b660aa8c9ae483d8dbaea4f97692d74", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.214.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "730ab4a461d8c24326bf8216d6e9471b4839b421963eeeb8717f1258f69d3ffd", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.214.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8f52a13bf900a11aae91c04224c4477cd9b5839374c90e7ee63788920df4e7a1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.214.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d390ddf2c579186037ef3bf83caf328a6400371d3a74dfa48028f786e5559d6", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.214.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a86842e49be3f5783a9f4b7c9d2a785a6b13b9437c87458bd8afb8d1a2fb7f7e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.214.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "28f452d3cae1a6cde402829df2bd66fd042567ec5d3160db1de8430c0da820cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.223.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ff40d035bdb712448418deb651780c46c903d4c8cdef8981adb8a16bfc5a754", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.223.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "539412fd318292a08b15e849677820dd4e6bc0c53c154a4cd878626a3138b2f1", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.223.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d959e835e383e54e1c5cce827b547ca90e172609d8f19d2b8d5942916c05d5df", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.223.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2a37b9018bceaa9bb10474b873194aa86b6cbbf55de7e59745f0134981ae8e79", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.223.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cff01921b18273c59ffc6fda1d67cc707ddfbf25ed2baafee368e9418c79cc5f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.223.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3cdd41453b57a4a8ff597a03d4d70ab157911a65523faa1b18a2d8ab51f9a4f5", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.227.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8451e884797f04aa6c97b5d817503f1100201949b035784ca9c38240d81de5ba", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.227.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d6a1107d5e77556a5d3c03250f8ebdddba4a31ca9ed264cd5c131cb16993d308", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.227.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c838efa3852e3440e04b5b954fbc48b9b282c2749e4ec723357b2fede87520c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.227.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "232665d34546feca76b6df092c2c683b731dc2fde41c357e1ced8977aca7bd4a", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.227.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5ed3019c57f56f3c61c5a68c40fc43b95c6844455508ac60df4883a2f0afb17d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.227.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "311beb6ee3cdf8a59a3b679ad3587afb143b9c3d8129a59f8f3bd1345889b6ca", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.245.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a343aec079209eaf33b0b12dd8e829d8f13cd6688b3721c8bcf84befd2a745f5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.245.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ac48851a0c6cf77e3327364cee8fe47170517b77d3c0aa52a0aad0e08ae8b700", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.245.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1ff442dbd33b453a92499fdf38dd9847136c06a41412c96f72a5f757a96533f1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.245.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cae542930054a9314ad69ed2a3947a073dc27916d4a187a818a829f9d9dec83c", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.245.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6cd7b2629ede161d71d4d084d12d287b4d42fc155fb1d0643754d91b6f8f636b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.245.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "956652631ec75254e994123c2d55797c8cac0c4d8fb248794ba56f03ab461ec7", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.246.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1b5463880aaaa1de46825a1104a6475defce70e315ba47da865f0c131ef3e7e1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.246.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "583e444a0fab7ec542a771e6463116a41cac5652c45c0e3c16f7ef974f4ebc7f", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.246.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ffe39a8353601993f2d2b00c8f3ce889a4d5086d1b1ba3699149b9d1085875a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.246.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ea52d5d3095bae7ff6aa49ff087d86ae23df5820638de65a0ac384c9a4294b6", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.246.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c65cef1d1533e441b0b685d8a6b736f1aa6a9d45fe25e874cb2e04ad93326516", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.246.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dcfb1c2576417a10ed7ef9306a57a8fdb46bb4ab61a5399ee63505112d8024b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.248.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ced9b23d6b977c183b164f99f785c45bd76843613e6851ce126ea00888c50164", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.248.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4e1fad72cf62cad49d98a525eda050772b559eabc9146a79a02deb1394599fbe", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.248.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f9c90e97d1e73f8cb5c5bf0a05b030f599d6a8b04cb25b2b027c0f023c6afb15", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.248.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8031d192995fd1f3315f802ff84ba2d226d4381c4a3bcd0013be3f9945e0dea0", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.248.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d09b72616e0af21cfc289442c71b49bd887ae18eb0ac8c8fff43f0b946178083", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.248.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "61b6256e277ddeb17787fcf40383643bbcb8dba5bbfd2abddf6aa65390fae4a7", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.25.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8a9da17ee25f21df7ca952586a94d76a670b97f407897cdbff7e2981b4c9c53", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.25.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cb8f9929fde7295ef99223daf126d58616bf7ed6600248bf819c6d1c3ee4f577", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.25.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe999f42fb05292e2ddd6738a5cd96d753e1f67c703123cbf63ceb62d0e5b602", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.25.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0cbd7fdff966462f7e2b9eecf453471b13d36c791b6bfc96697868994786e908", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.25.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ea9df62edc14a6f2b8ed93c07f3838a1dbcab375bf2b033b5b400dff845b8205", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.25.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9b83d1c2065c8e08d8aa956986d3064fc146a4252f6876dd987d55557db9b531", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.43.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0d202f9c48f3ff791ae185cbd251d460225d8110998575f0c851c6f7688fb6e6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.43.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2ec719ac7674540c9ebdc5405a00feabfeb79c2d006dce8c8624eec27b23f5bb", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.43.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "39d6fb04fabf7d530debb8f192ae72f38073304967975bd5dcad731b25d8212e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.43.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c9daf07a255ea05c4bf323151f27c98f6be4d87bfee82b22a3eb487f6487e0e2", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.43.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8dc66a8a72a7ee4e3b4780e74615b3ca154aae46cfa90aec3b779c3ef3591646", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.43.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a922111f4d06c20665cba0bd67d0eb85fd7bb0a53760ed2eb65bf38273c49bc", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.49.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "28b248ee9a5c3e80ed24d9a917974dd7ff095e6999ab91ae2099bdbda286914d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.49.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eaa9417b764ac1a491f913469c6610f608ea2d4cea976ff0361b0fa2415caeba", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.49.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cfc68fc8515093e53e9f578fb16b55fa789934e6ed996ea6efae7d37f30bb3a3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.49.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "147313f5e977773fab85881a565b2746bae85f09276289eccb18815cab6cf430", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.49.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "60eed192d573c4d15da09321b2e4a9b8d769595fd3ee1e2ede6adb4423f11039", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.49.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "79b1a937ebc02e33ce276a387b56466d29f6a3e516209e315fcd6d477b46c92a", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.57.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "122341d1b731b8d3ccc3753360a851b9ae73cd97bd0fd67c3f0cfc28d6c53ceb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.57.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2ad68a457fd48a27747ef976c35d5594479d2e93ce648369fc9d9cee8b18ab71", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.57.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d55c4da9effa810d08c70cf6bbb55bcbb9e0794cf91b631c7b6b83a6f363680d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.57.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec9804163d68e49360dab9c8c6b3d95c041de623325d01a3274b3eecf92188c9", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.57.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "444a6fb111f3f4a352f65b52b2771c82e29bafc228dcb015e7d46088195e28be", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.57.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "702858c1fd7e292b13c39dac270e65548660dc91794937452a88117dbbab9481", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "61f894472520a5c972d9dfc029e1f98b850fd7ea4db83b146ab6a090c181ffc3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3590df81512853d7b210163e3f0632094c6d2922ac0dae1debd27aad10c2819c", + "shape": [ + 32, + 16 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32be383892752dc924b71dd196893440570c9f40ee53f9de3a87430f8d4d091f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6733d972940d90ff5821dbadfedd925113c2fd61773166a6b7452eaa62b5b304", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d8105609e56b62e20dd83cd2acaa1cbdef50fe5836d35177df4a82f9254c73dd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.26.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b239a5d8a5ecf43b6914a46f980794b26ec1b41b3d4dbf4401f849cf139956e4", + "shape": [ + 16, + 32 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.26.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "4e9da4cba530d18173587595e9621d665dedcf72c9ef3a0d63adb453091afe35", + "shape": [ + 288 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.26.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "f55094049c9acef31dedb63087153a533496d3d0b0d530c28dcbf9cbacc9ff9f", + "shape": [ + 288, + 4096 + ], + "shard": "model-00026-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.27.mlp.experts.106.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e2645f1c89fb98d08c07fd0623c8a8ace2578662cc3e24d76f1895e00ebf79e2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.106.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a686b502abcaf96da6d8cd195178587ee21465b665ab59b265f2ad5a61d0f7d9", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.106.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ca73adc1ce346134c34cf88df428efb0e66a1a903e1170220f16edaa13d219e1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.106.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa02990771a688dacb67f9f501c9137fe4004b1436311fc81ae5db7ed5f3fb06", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.106.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f29338684722995d2ada34e2df98d0f93c3bf803e68cc9afb6995460702df016", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.106.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "feaac7b831c20ca1d5e96939f7e67ea208cdaf88bc6a7c78169096a8cf6f2bba", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.114.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df8c361a380373b07eb983baced13eab9afdd173e973a30dc59f059dac1ed25b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.114.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "46e65ebb74d93c444b5941c02ac87b7c70608067cc76867d1fc510087e446ade", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.114.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1e26e4295ad2d830df30a65c6133f0e4400d814168068adc4a082338911f51e7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.114.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c74a2cb61354bd50eb26b7c2a1f9d6928af0ed8a8fd2a66951329d004ad26f54", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.114.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2adc46aa7fd33df4dcbd8aa1dc881ea25640369efe5b58c44f662b1a4bf3b5e8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.114.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2173a3e4e19b45e17afe08854fc9c60c1dd678125815cececb839cb3c41064ac", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.121.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cfe12cc37bd0eb76435fb32879d87b0e66e3ba73cdcee63d83cc3802649d3452", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.121.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eab273d4faa67953a3b84b0c4accf8d70a74d629932d42211c01ee2bfb11999c", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.121.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b568ddb3b29ae5474bd00c658758cdbe5433a388428abc701172ec257f339dbc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.121.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "030914588ac39ce351e83f55c3e5d337f2f52527f8fb5bb8944b67eef810a73d", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.121.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c7fb27f3c8e160ebde6876c887430ac436f62d30c31af88fa46cde209fb8a68a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.121.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bed8c39e93b9f1bf5ba5e3e9c4107555c38e1f6fe580d3503c73f75452e029d1", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.134.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5551bfc711392ae103b4af726782aeba28882c5d0dee48e0308d7275db56227c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.134.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e5af159d60c5af2c5b6b227904823d8cadc454b23e82cea4df1904e0f8a58191", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.134.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5dd6bd00ea41f4a9fe5c25211b061c5938e79e0db106fece9f7511e7f128ba8d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.134.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2862635e7b219ec9d14be30da5878e49c422a0c54f47876d9ff7d7e636c1bc2b", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.134.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e0abfb58ef8674027550f6b3ed3b7d7c56b735e33b8338e226908bc4326b84d7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.134.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "78240b9c9eaa58b9f55f06fb4cd4b0935778d602fab581f3171ccb35b63e93d9", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.188.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5920680aadfdae88a1ebef7b54c144c73f442e5c24386d00bb0adf3d9feb94c0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.188.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b1b863c1de4e94fb554e819b617c66da1976488fa1a4b2421d2df1ae358e4267", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.188.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6414b92809c118e16a28481a69824ef432c884d909ef25730d181b617c008f34", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.188.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b2fe371cf231d29be8afbcf8a2bc61fc75deb33fdaf82d209b620d59e8814963", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.188.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ed9355524a103c7312942b73d1ffd35ceb2cdbe2761d98228639624797e341b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.188.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0b8e46cbe0bf486a07c8d57e2ae0bbb54044a3d165b36449c19cb2ee03828cb", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.194.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bd352742f46e36b217b3a88d02614c914ebdef6e602ba5192b2ccc3825b2e396", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.194.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6075e50887f163f69e72c9ec79ae8f3b11a0b744deb35c2365ca805d74bc4506", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.194.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cd6b7eb87e92617ca81e69c76993a7bdcc7b9edf567b16a31199424011f4f025", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.194.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "02fc5bc477aa5fa3d332cf69f2356a5eae314e941a17c280c97a6c0f493fec97", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.194.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "85f112e052a167c86170a347d08317a7f4d5e40b130e4cc44350092e0d1f4e15", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.194.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "99d9282c8123ef8a5d4f7b9e9530b3d1f8468ad72d219dab3924b862d885bf69", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.196.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "11c0c0a6572543c150e5235d3d3da41e1317d4e7a48b419d30fbf809bdaea064", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.196.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4349530f7b0d08d27b0d644c7246d5fd99f183ec849796a9c9b85c4f7843a91d", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.196.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4689c5ffa0bd982b631bfb1f1d7ffbab7138e8f69c424a4c3df30c683015f707", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.196.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e409fa9df07ff0657cae882ede1b8a2638f16ab3d2cffac9335a69b229795360", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.196.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8dfc7cb00fcd4ab61ff9b2736764d08a914abea8735749c0c2b430ea4319b979", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.196.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6c1c68e2c00dcf51c5e99cd457bf4af5f13902079aa9f9081ba6c13300e67579", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.205.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4ebb4430ea1f95b119969f0f2a82e8214e3e6e05de8272a915c4f985e6b99754", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.205.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de8c896795c0b4580e1b21aa55c7c9a9978e303d265927188ba65740fab376ef", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.205.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ed77b9280d1f559e48d4be5bd7ea93be1b9b440a98be3d6652d60a4450c0847b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.205.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c2fed7405b47bccd58f32a71dcd855a19213c1a0328e65e652ed82f0366d4ec8", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.205.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0eb8ec78ef569ab33ed6708a17d1aa9d42159c49caf11394cdb7cbb13f689ef8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.205.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0075c4edb9e4bacc1645d425b040815a597316d5a798d6b2088975f816e2ba76", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.209.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9cf32913c1cb05d79acc48575b5c99f3eb245fe8475cf91c77e6251a40749d73", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.209.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "51dca8553ffca4579f5c2132876c4e7b5a0aad8e30ff039af272b1952a35eadd", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.209.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "45fc6192dc83c3ab3ec9e1ed370f1e49f6332d7909a2ae085a926bae4eb03b80", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.209.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e30a315e9cf52483625e110e13e1c0bbdf107486dc805ac1edf7c60908f27e6", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.209.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4cb42ff60615a9d685e1166ca86c6750586729b9d7510fb2d31bed41353178cb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.209.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0ad4e6bbe5d61ec140a3a6cb4a07b80b8659d24a73854b3da21f78977cc23a65", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.224.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2cd27a82f2e4de03da8700f20cfc9994dfccc2a49baeef1b9b3be1c8e2030407", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.224.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ba4d0f7c4f2f0d5fe78d115805642cec738c5966c45a259e3246922a989bc2c5", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.224.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b496fce69bc86f76218c3a0ec7f83d7b85129eceece1eadb1eabb9652605404e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.224.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ec5d4a9e9aa77f9afb5a15fdf3477e48897042f3059fb341eb85d299d004596", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.224.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "92b78ac930e12e9f6b7a6c8a0f5814e7cce8c6162c9528918802ecf282201df0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.224.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c247d5caaf37973cf855e4112ccdffadfe238ba63a7311f11274c023d98ad042", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.248.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cbf02ec2807a039833a03166263eecf159f6097b2679ef000149199ea2b9a063", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.248.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3586f42b46ce113323309694cc2f07139c6f4d852be77a8f20b039b65d09510b", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.248.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "10d1e8d2d1cd8d43812f9caf8a9cb251e1461fdf682816fc2e6f7c39c5b6613d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.248.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec153e6dd31d57d419f1e5146eef4fad1790f0aff450181edc49f2b8b7eded4e", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.248.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cc2a9a2b89b0175fa2b9be60de3107a70d67bfd749bcd07aabd079b4aa2f8034", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.248.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0d52a340b00e1eaa0701ecab0ee26583cc7c6399fe13ed28a613c16657c0db41", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.26.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18121a6795f28f3c75ed0e09d85a61e6373eba926d7c75dec860a2b55260054b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.26.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14289a0b5bb0983358c571446447f8ccba5f2989bb1a4d35b682364373c13e43", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.26.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7b2a5d677afb92d388d87dd9a36a2f7c5ade2beb2e57e2af0328fb941345188e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.26.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "888bd77c3ce68a18e089e3ce59ebfa844ff78b9023ccdb838d76756b112d31d6", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.26.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4d5244320f889b8c3232ee8c752c84e3849dedbb89fd5d914b770d232f6613c3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.26.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "74e62f5987b45a74b851b44b6c075260701c1efc0ce69bbc639c9022370af0af", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.286.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "16a92c971b39acedb4228e3f904f56b6b34063ab79cab876b773581aa46e96d2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.286.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f1266259ba90bcb5234d25f71c9b74bdc311cdf4b058ec472b032fc5adcc4576", + "shape": [ + 32, + 16 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.286.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f75a4b7d2c4ed82ba217ba5823f7b288d5bce29e628258017c9eed3693b1efd0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.286.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "612be8ee3b27ce8da2052f9ad935438568b7d6ca2d5aae1ab538bcdd0a5adb25", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.286.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5c2184b5a070e7a312fab54ef8d5c7e89756b80746e239d6ab0929e67c6ac88", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.286.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b539d51e98cffff208a8618df29597044918659fad254c5aeb6ef732702160d5", + "shape": [ + 16, + 32 + ], + "shard": "model-00027-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.60.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f0e82116023ba3d1a116bf4677c21dfedb41a7b23337d59a019b022c96827db2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.60.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df3947aecb6b8d001617231717c0c0f27d82561a9942bf592f05eb088abd0ddd", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.60.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac417fece4b36f77c474cb79a4bbaf10a1cc63461d37164c1c0e438409998acb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.60.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "850b5b26434d35b6c58324e02503c8952f5a4db932c8a2ff45c608e091c566b8", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.60.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c7bb2ab630688cb3c65fc10b0c1d14aabb93a1c2bc676956e5d3585cd74750a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.60.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "89ab75e324bc27581381aeb52752da27c651a18aa78a111afa59b78967ffa814", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f919888e73be9f43e59e079e742cee62e71798e8f16dedeaf8097c44fc051300", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "33c44ba25e39793380f12a507248c4d92e1c7473ea10e0199df7531309386daf", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b128545d3b7898e4d15523cbca08e77b1ec97545f01cc35a62b6a7f869ec0529", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "72724437453bc4c2578a42e15fb6405497dda4b1e8f46a5425c11416566eba59", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40514555dac91141a7e5af5bdde86f9b6d5f75790bcabb5df87bb73965d1a126", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f70cd2af07858db1402fc107c610a8581dc1f6c6305d1f75ea93fca5bcdac588", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa782e5e9e4c4925d123c52a182fb019318e62c92241b04566a3bbaebd01ffdb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d9d333dfe906219399aba89a95ac776c923816b026b78f258f6c671af1ac3b7", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e54c3d4a80f3ac671991ef2aed0bf8eb8884111e66b168ead7d60823f6cde4a6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ff1b7d872373b1ff30b01b127cbca0bd8f83941fedddfef2ef9b17c3bbf86856", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84e829c3f79cc37013f4c70c26fd3b71a7a3f776939d768e1fc91d1b6ae0ece9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.27.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b356f2c170220eadb652e6f4bbff7d1e5980b35bdc66e75fd905907de7920f24", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.27.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "bb2ff4cfd50961236e189c6f83adaddbd789a77038cb099aeba639fd41e2a62f", + "shape": [ + 288 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.27.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "9fa74a74d3f4d49703fac0a348a317c8716355fcd994049b2c242f0763f79e54", + "shape": [ + 288, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.28.mlp.experts.107.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c870e95b8f926722a38073c9d08e3d1edb31a382af5763d0722334d86971d11", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.107.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "442277bb860d9a7372b0d52097a6d5aef463e896ffbf4015375c17835d2953da", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.107.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "daa8d2c061c829c238a2d8d4321e32089dade5c56a14dbd3785190590e14aeb8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.107.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6fc3cfc7f31eac9c8ec1226e047423cb24ca08dd9fc16fe36d354c9abeb569cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.107.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6fd18be6f1d549fc116be7a66a0a41a80fe2f0378e5fd0118aea0accb501e2f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.107.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "064da98afaa59c13f1e2ca7dfa45bd91bfcb0762a5ac6c5e188a08d804c324a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.185.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a4383c25f53b52b369b65f68e7329f4f223e447bf5eda6aeab9ddce278a4662b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.185.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "68b97140f3df2c309b0a0b6e8f2b0342c11f893e3ca4b2f1c5205997096fb629", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.185.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cc9d46c94dc97559f29bb986fc31a242185b8e9c0b08594132755326cd2f6c8c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.185.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "75e0b84c5f45ed84b321d00f67a0d530bf054286b4e13e25f14f2842128b7bfa", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.185.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ca42e778976f76bdb3d049e7403815978cff224199aadc742871396aa8c599b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.185.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa8c61c5d36420ec62a98ff8779cab7ff87c9d9a34e54ae707e15895e6b46d3e", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.198.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d28e19a2c5d1fd2d3ece96945e33acc7533c83817d6418f71d17fd8378969fdd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.198.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38dea736e76101029567eb76f635c9ef8d87fca636fe11f580fc76945e1a4576", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.198.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cc7b73aad0ac2fafecce56b0549c3af019f921ade86667e476b66b6623547cb5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.198.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ba7beec15545f7b78404e22205906c33b4b9aa4f88dd7e9ad85a0551f055afe", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.198.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ce786d8284f44c76fc7dc581ddace777dfe01b8ef6ed9a96982b175e1a01319", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.198.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c0f3abccfa695dc1c16fbe670e8800de1361b12045048e23ad7794158f965aa5", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.214.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "85d88f408a4577203c52a050a12f3f6633b9674b83970ff963172ceb7be6bcc7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.214.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c30ea0ef18b42adaf297c9db37d633c5e55ea2678fc64752b294e3c1c3e45e86", + "shape": [ + 32, + 16 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.214.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "342ade9de0a32b4bfe0809b58cbc4dab0fc68ea9c1e4f9ccb4096659c73e26a9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.214.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fb8824a07fe4df9a92bddaec4b34a336595ade193390d198068619a20db0b7f1", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.214.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6d4d9e2e89799ebfc1fa0b29f2001eda83455f0ad5b5a7e3098dd151751ef7a8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.214.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "489d47806259e4329b1a0888c85c942df21ba79233b9f60f655cab8be56d55e0", + "shape": [ + 16, + 32 + ], + "shard": "model-00028-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.245.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b21f91a243a13ef18117134c11a4ff9243433d072f393b6421b6aa2efeacc1ea", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.245.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "faa8761c2ff868bd468f2d04028c19b3a358536319efe54bc8b71ed703f1fada", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.245.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fef2957f074d3cc8bbac35a598714368bd029efd74514a14e0cfb122938fabb9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.245.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b65056d50a03306d3c7fd44011ef3ead86248f10aa58fcab4295243dd4c0dde", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.245.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "909451900889abd8fa6d7d0d732fced73b77646183e89713f328401b0c89d446", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.245.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "450be26c7c6536f84a28f5ddc15058cfabd56279f15ef11a3f7a190298571594", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.272.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d0a3210482187051b533c5ceb508aa36a2c351a565c7f67f40fe99c7a07f7f33", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.272.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "35afc8ed5d01409382a56f8d7104eae421ecd1b96613d2dfd210c83039589631", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.272.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b5fe1e6cbcf84d6218279718ec8829977ba606854412c5f3a9c70d79c717c1c3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.272.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1e64ad21f9b794b048951da729677ae79c5b127581542d440c9d596d5dd7847b", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.272.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "52fed19e5dfe7d8ecfa88a1803c466423085dd1a288fde2b963553b1af2128f9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.272.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "904f28b0e59a6c38bd295f998b2c888200a511a29a5f91c7a9fd6d75ca09c4bb", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.283.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "710d607e679441cbb3750fe493490b9d2bedfdc55bc594b8815bad230933cff3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.283.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6499789b8edeca11c333eba0464b06f0982092b1ded65d95ae42be8342d9228d", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.283.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c81cc75782286dfb6ad8e996ddc0e960d9b863c1b786db97e6da957a756a1b4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.283.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "039bd7d7ba445214d3bb0b09a3c98477d6309ffb80cee8650ecf0d9cb535bdd3", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.283.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77a99e4239e82238d63a788276316f218aa0ab280d9d24811fedddd3e0ae2237", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.283.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e16171b6a11e0b6fdd3094ea6c53dca211ba99c7f4db4c10c357f037dec38ad7", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.50.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c4ccb877ce10482447c85d1d77ffb80a755bc6e53fe5538e673f5ef2a739b698", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.50.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d7695dad861c3192058838aef95b2e32b39d76242766f95b0535df37bcaef7e0", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.50.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "489908e1aecd1e63e4421a0209134d78a61af0142d92758cf661b90295f4f341", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.50.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b8dbf340ea71a306b64f03bc1776c25ec9b216877bdf47d6117929bc4b6c737", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.50.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7cea14e5842a40377c3705291a1d903d49592f436fc1f319b185818c8659b721", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.50.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c05f22971695909b04ee4109840f0f0189b635752ac0a08c7242481ec91b5fe5", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.56.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "460e1712a4ab39a44f8ea9a54214265587700cda40c5d8298de787bec17cb935", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.56.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "09ab56134c4af718c23c20623608028b04889ba2a3098be19debd6d2f5b4add6", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.56.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3fca4bf43eafc7a7988779bc97c42549859a8a22f0d374b041cd49790c7b6124", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.56.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f309cfdb2a3816531c64d1a898f63e60f0ab930dd98a1a39c73290ade88002b8", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.56.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eae3d7e12768d3791b42f191f09091e11499e144db07f621e931e7f428b19f69", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.56.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d6715c241b14b156a2b3fd8fdbf0e0bd93d222f31128ed356e5bfc1bd30aa82", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.60.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d71cf298fef62e2a0ecb5ac504b98ff5dd8c1b710a4038179d1685c3f63ccb15", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.60.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e48db96dccc2db11c48eee3701f384ad60ad78a3be0453332254681528953265", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.60.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2a3b7eadc28c3866d066b751f6b1b42b0c39471f9356bf3894fc7d78bd27cd5e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.60.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b26c9e3dd04eeb7d724e8a9d0b6bdaa11e27976f4fd77c98b283aeff473fae6a", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.60.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6794efc3b1c952fc7af163f3d72b2f3a52fe723d9694e226dfd30d764c313583", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.60.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4aa41ec64e40429db0eae6d48e09cb05c826d5ace79c791c6ce424808aa04ebb", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.61.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "371b74610a08ede3afbc2bc3c7a92f5fa6cc6c83bef87eb5d4df9dd717f96a89", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.61.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e72ca5d7eac8b8e676375d263a0a7cd793cba69b1d265a821e50c81636ce5b37", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.61.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7a1001adf1e65332d71079379ccd8ff785e16a287a819ff9972cf1150de239a6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.61.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b12cbc25fbdc88512daac44b8daba1d24859236911e7cf80c9d3b51a724babd0", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.61.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4a4f80762e1e72b263ad48647d78af299d5634461fca8230c8e7bccd2c50d8fd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.61.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16e3e137cb2af7d3e40ef43f2267d1a46e5f4e5c9d874226263bfa450f344bb1", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.67.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c4067b119125300381485500a94238b135dc0c8b13072d8cc4d11c646a4024e9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.67.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6eae5e34a95fd0bc2a5640f393bb878356f1d1e4a4f8745f650cd097bb22aab0", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.67.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b0f22b661c9b93842420db67cddd15da7332785b9ae18e8664cbdd769a8906cb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.67.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7fa4bf6751240ee6175d47a66333e9cb6826caa19db473d12affd2e689d73e4a", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.67.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "af9686d46db15d6b3519c30b6b5187008b1d311f2e5c8110a23f368405f7f559", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.67.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "601ba318ce0b7382b26428c56ea9b49b1f842bdab3703b12909109456b8d7fe2", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.69.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1b7b555c50ec6161a484e59300ab167769e3d674d54793c15d47e3c400dd6fb0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.69.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8e71abd1bcded3fc4e90b7220ed30f545af78fb30d8f2b0982c5a0d00973f7e0", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.69.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9d9ac5e41184e21f7f61e6488b2ab506f4c5e052d0899052fbe3f00cdc4686bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.69.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "698ced92725262bc10a7d9b4675010c688e2610f0c392e79d54e2004ea1936cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.69.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2c89cf8282e1e962dd766dac0755e953ac1448d1d487cd2158c9d843d5806665", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.69.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8ed2d24878bd88b7cd6311999c7de07f7f97212811acfb08a93093aa95c18737", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b54c95a1d76d53c1e30e954c27bb8918a849648bf6bffc4688230a559a30712b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bbd836cb363430823026ad30e23f9062b3952a154643ccb281584bbb9fb428d5", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "388a14767ce35487b809138dc0a0bebd0d2b384d2602afefa551c3da660d2ee6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7f5f6d7df9678fd8ffc517f25baa839bd3fd4436f293ee70d415588389d6d05f", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5b61577065aada4219614f5d5e1667d53ba724bf0e090b2e98c980c994b58a17", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1f7babad2d85357383b54273033fd63813ad83bbd3e86ac28780341cffa0192d", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.97.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ecd2a5f37402cf7365af507e5d9d7771b4b240f87a5114650d2270a32ac8c155", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.97.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b4fe6c5d586a4079aae72da2415ce9105751b213706cfaada7281d739939d863", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.97.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2a785200f9f9ada4bfe8b5833e6639b0f962f9c1e5d97663aed6e33c25ab965", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.97.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "011864ae1da2613b9f6a0b78f38835ea3414fa279166f6b3a3b0feacceab9aa8", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.97.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ec750df615544402dc0172a9df718922a2ad68890fc535d47cb37584d1637211", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.97.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "10bf3f2c395a1bd841a3010cb9dbbd56036875483ed0d370c44f788a8b4accf2", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.99.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "98d061049f048868c998b132797ce10c1860b3760c2c5548aae02dcb57b925a4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.99.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d2e979a4925c1139b7936ba0dec43eb8f4667ee2c39a158f65181bc89f5d1de", + "shape": [ + 32, + 16 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.99.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ab97a0d25f624edec8be061a7c50a2e00833c2009bfb11704a60f71cf7b24615", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.99.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "536bddfb97d30d9388ba108ad9df30348f9fd3b4eb15d35a208e71baf1b65915", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.experts.99.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b9bf6e1382ccff65da303f9a77ed991f4c908b15d7a05af06ec0f13cd26cbedd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.28.mlp.experts.99.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "70e4537fd99ea47e10766c29ddb11bdef0d1651ef00bd326d0826ebda9ac167e", + "shape": [ + 16, + 32 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.28.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "c9a8ec21b6a4a5a7b1f8fa94b25a039b12dd4f987d2af3d377f7e1d7dfc367fa", + "shape": [ + 288 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.28.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "a88c15685e47bd2bc957519006143fe355173a8ac137dc884406487fc3d6f72e", + "shape": [ + 288, + 4096 + ], + "shard": "model-00029-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.3.mlp.experts.10.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "279b617bfabf7185f8f5deb5754f10ebf297c7515d5fba1b1103dfda1c479a93", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.10.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9eb518e1a67cd0ff140d0c95d397ff6b57d04d09e3d56a36303d59d4384e0496", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.10.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6c21831efca80f7a5975a103dd6de30877f191a81f21f69e728599b69dceaf98", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.10.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "efdb9c63ef62db35fc6417ba4b8857569d082af1767a64771588ac9f3b503804", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.10.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6d09af95aa3b65709cc161c78c5cfd0d38e79e754eac3129df46133f120fdeb2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.10.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a420ce8d2b1d83942f0476dfbccd4fccdb77a9d320388d6c411b91612d10e963", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.107.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "37039e5be344ecfbf7ba36cb03859d4cdfa46e09f74b053448c31b134305f193", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.107.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "137a38a3754d923d35640e71b101410ffc4c9411a47c48b7fe6be4b062228178", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.107.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bf10944d23bbbd298a454e9e461715895ca7b85300486f75690b975f4c115bc2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.107.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "584c35281866ec3882e53ba997d2ffa177349415b85485c2042459d2a178226a", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.107.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5919202398e0c71d56e0496874f679870dc9cfe3a7d32223e2f627887360d5de", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.107.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2512ccc46d1d6a1007d65c7d4f73ecd8c5389b7ec2329117716583765a631180", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.118.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "be5f83d1f5ae2196ce966cb7a1c978dfdeb7992e262dece20398680c60d39979", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.118.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6d2446fe99028114387762500fff8bcb963cecfa0660a63c094cb3147b03899f", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.118.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "421114f09e2c6db37f7ed4d5c6ba757d68667b6fa283d6bb444d5c9b7047f706", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.118.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "90b53cf863f0a34a091fd0de00c459412764531bef6976d84d94d43744d0d847", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.118.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c03bf57823de6713ec9e62daed6697270dac0b8a42e71f3bad9b4335d223adfe", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.118.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5881f2530f9f9a2c85597cb7ea7ee196a223c12cd9684dfd47025f416db798fb", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.12.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d689cca78229beac178bd57df90d3fda67ed73be5b2a721cc396e40735086520", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.12.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7601911836d93105c527f75d749c41af53b594f4542afaa01c902a17a2a621ea", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.12.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "95f5b505c28052d58dad2b151a75dfcc5acac2f770c588ebbbc91c2370820c34", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.12.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1200108ce70bbfc0115e99b1b3c93659e723bde6a23cf25172d4b0200664a5ae", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.12.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bd5930e018ec53c9833ce1a60dc398910b8ec849cb11d5890fe3c603d448d117", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.12.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "116aac564bed053d1ab9bd3d8ac164ca39ffaa7bfd187c47fd816e5b3c62d7a2", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.149.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b7ec2f4c3dc43d3c48c62fa2885b2236ea6d52485114cc548cd8d031b4fe9d96", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.149.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4297a98a746f46e70b01e51313bb40a8502f2ca59cb433d9b3a008ef86070126", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.149.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "19f4389a61707aa09d6108c6b7a0990499e68443afae7aae2f81b3438e42bc69", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.149.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d9e54cdbcbe1890510240ee8c6f72db76cd4391d96828ebf4491620c20dbf862", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.149.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eecab2663083a386c0677beff927b741816cf9eeef6082d6c6f678bf3af2d6e7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.149.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6c03faa0443d5fd8a0ae1b8739a751a031a9b6b6f63ae649afcb7a341214c8b", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.171.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "908b8ffb42d2f67ddb90a2048339b173c9c620cca8358ee59f0dbae7f8cc251d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.171.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "34e754c96a36c4afad9f385901432449f570f40ce6b581b40dbbc24f8dabc721", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.171.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2d29fa227dd12239978dd095b63008ce48dce284a5cb1c70c5d3dd528866478f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.171.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "930828230f683f4c1c64960355d9ed5913f41a66cb09392cf898541634b70da6", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.171.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bee468e6180fdc4d8a0e5b0044b523c1b1d0a21bf1762bbca5ee437d56e0291a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.171.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "53a564db4c63195d2c2705c059de6534ee117fad473420a1127a9c890d3f01e6", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bd3207d318bdb7cc7d2f892635725901da4e08e1a203e6e416634162469b64ff", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ad0554dfb996788352193ce855931d6e80a825ded337a6cf08b400c73a03aef", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac16816c211f57c4de5799bae21a0fc32b97c1efabaccdc5b700414ab0b6e49e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3bd3532f80d52d350f53336f82bb02be764965a80004ff22f0b57c1b4b71ed0c", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "494b6cd159a339077d647af618b252e4636d91f6a976c54cbdeedf48309bd73d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7c0c0969a17afa88b9c46d0facf43701db2ea3e882fc6fc0ad1f0d5fdf4405c7", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.186.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "16e2246af94cab35aed4cb29c7141e9640ef71a6809f3fe1c0dc71cea87ef909", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.186.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cc7d7770cc535b58fd3c0753c1c06eaf4740efcaf213bf58fbc1e3428c8627f5", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.186.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "07e02194f139621df3d2619a1a3d28248c26f4aca97b399fad3cce8ec554523b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.186.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8b03017eb8d84cb13650438c83a12bfad53dc75a961cf092f5cacfb44c2eb94c", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.186.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3eb84fe8a3bfb3f6b217ff953493b59d13cf9711d249f3dd817d83d48dda5d65", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.186.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "85de11758c5b3251b694c3eb29a04bdc3d509bceaac4e0eabf871edd5396655b", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.189.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6716673304b9e88dcbf11522d0a4069d281d78c7f1c0ee1ffd82fe8eb9b575de", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.189.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "90c5d752a58a40f743367bfa41f112359ccc2f09cf43805baf601de49626fbd1", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.189.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2928f01236ea18abfbe5467032c76aa10c5dd00b54b9685f4b579ba7af3070f7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.189.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a61549101a878c427eb5c05fc9230cdf0ce1580fcb567d32f2ebb3ee6d82706", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.189.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "122f41381c5d0714c5e68256552438d201a14e50493abbf4574bc55e76718535", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.189.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e60592154dc2a1ea46f8f7403d52ff30bcb1a3630c42d6c1f85749149c030526", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.202.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "83de30d192af11b1de364de4c871077bce008e85f5bc3ed03341debfd419fd08", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.202.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a83a0d34fa7f9fd23f6e3d9484dbcf61f136d4c27ab1d3fad96d737c26962776", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.202.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3980c094286e6f4c4b5c148de9092d607875dc405adc0b8286b8301cbb7e8ece", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.202.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "458252724b36aa84c5f98e766b2852d9ad830217f2a34f52d2e07a845a0d91aa", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.202.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0365de25f1d15b8e75152ba08a9b58320bf2fa49dbff69d04b48c3743d0a0134", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.202.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2f47e6940ff1d67cf8a902dc5e53bd51fe21b5617d30f2d38c54f7428e9d079c", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.220.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4ad12881882c3e2c3233d86e125fc0f389d221c6aeee8ec6b49ee50516f26b13", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.220.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3fd971b49517252f22517db6433199d1d78ec70d4a588fd888e6db6189e41799", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.220.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3c4a57942b9c1114c739deaab31a79493701035ed005e96a1dab60abb3862499", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.220.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4f222df73804edc114dc242c048d5c0b39bc1a1f9234ccfb45f8144b2815ad3d", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.220.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2a7e8f4eeb488fb2cbde731fb417d7e01e86f4ff8591a718bb1ae707181c9130", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.220.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "65add2c1bf9d59f3918922c118ae863ec23e19543ba2f7476488d6d9219c60f1", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.24.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3743a2122c2c05b49f6b98a305a267386f201707708a70c238eeb5e3d0df4683", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.24.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "171b4093c4a39af323dfbc478ce4092d47aed9cda516f6ea8439fbd29d471a7c", + "shape": [ + 32, + 16 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.24.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "95e21386d1187e1c3a3abfa3c862990edc97557038aff565688d048ebf281d65", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.24.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "03ccbeabbd7689970246c9d833a27be56e7571730223f0a2f37d9c0e6d0dd77f", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.24.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5f8e63146c87425a76b36048c98b5118c4dbf8015feb1a43ff0edebb9d6da224", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.24.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a9c29976ed5f056d4c34584ad001dcb5c86b0290043c191fdf1ecbab300d4258", + "shape": [ + 16, + 32 + ], + "shard": "model-00031-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.286.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "339527d2c30eb6f6a8fca3ade6b94c2709701a8e9c898f1fff9440cd9c659c9c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.286.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dedb6683e98bf8f25fb759b9fa7c3648d67ff17809c7fadb38b924708e321f66", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.286.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "398a793df61c27378838aa478d33c1284bd42fbc4523b974d54e3a0b3a3dba10", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.286.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1bebdd3755520779df64697c7f625e592a589989656ee1532b0ae00a1c9c235c", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.286.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5824ae941510497cbe482156e85da9d2769de0af34b3b696ee4a47b235c735f1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.286.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7b3270844ea29435a6fab41152f47174dee9aa214a63c2141b0ec3bb191753dc", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.47.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa3fc4eee2779d735d45daa6b0b182a44118f723244c05cbc27bc92bfefafbd1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.47.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ac041271f8346ec4f1385323213d99c4249f952d1ca866b9c3697fab16999928", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.47.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a839a8de0465bbbf1ae226ccbbec1ae9981817567f13430580acdc8a542d7ead", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.47.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "85da8933f1f7fc39aee56af952091b69862853e058eda7d0a5e12aa7c7fa768e", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.47.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "19c5db58622a614b3359befcf3bbaab92b7ebff3808038f54ad6886ea6ce2a5e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.47.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d7cb133998a3efeab3a5eba2994cbf5e9eb4b28eaa51e803e033b8271abd884", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8092d157a16634e224b85888219a207aa535bd8ea24b7dbdd2af6a230e8d94ad", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1bc0e70e34b64484dea0c14cacf41ca0356ddb5e084dd4015b3920a32ca69d7f", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7cf8c07688d0dfb633c3867f1434469251612d785fb4a9ca12879ed47fa832cd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7989066b3a1d3edd46c5d2d77c14ae656b0c03a57c38b26e7a32888c2fe06291", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "597a9ce876b06b5d383ec50b3b3daf7384e06d2b82e1be673cbed6ba3c49cefa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "84adbb8dddf3d708ed68531550c360a9e68a88d643a3431b7a12a75e78e72bfa", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.98.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "777715eedb7f5108ddf92de66c43c251d256d721a3198ed2f99334dd230caca5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.98.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "54e5bf4b4f33a4a92fe02f70e1809d1300f05f8f67507192a683d61b6f77dc03", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.98.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "90f4322d7c0657409bff7a6e3987816f97f74423893278f47371755cf904b15d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.98.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b0e85c22088d4755e70ebbfeba1697fd2c0d79444176ef7040bc4823f7f23722", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.experts.98.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "41eee01c238df2ec3aaf5d824dfc5001b897a1ac3ddb1025ce041c66d8bdc092", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.3.mlp.experts.98.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e190cc0449b53c6b3161def33c72b98869db22dc8ceaa0ce3ea75ba54406243", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.3.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "f2234f4a90762ff1b98cad68acdf08c17dcaeb13eabacfdd85b2d9f81ebe0aff", + "shape": [ + 288 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.3.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "b4a11015a5cbc33595fb0c49f7609594b86061390a5781a220cb8ee7b524ca37", + "shape": [ + 288, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.30.mlp.experts.0.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a4ccf3a6acf45cd8c5534ac86233c8e22e6d6eaf027b68b956a4c16aebbf3c2f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.0.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "43700099f5560f1349e8d207413fa9ff27aa113fd945a66f5642ebb9b689db1e", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.0.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "22c473ed5d4200c0a41b4f00500b992858ce4f1deebd9d72220994704bfca633", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.0.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4857c79d07134f3b11af639e8cc3b84e82c9f37d9132f49c4406585eae81e319", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.0.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2a0588e2d593be6628779a1f28947157595a197999c2eebe4c0289a1526221fa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.0.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9e864db13013670d45ffa9235a70300b55101c8fff7dced274b522d931b385ad", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.101.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c3e8cc19413e998ce5e641e33c249876a3dd5b97de354337850d8bde8ae4c3f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.101.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5cd00eea9a28b18d69dd0e43b152d4a7cbc6285921b0f5aa6c796987f8bdd9df", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.101.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "94a841c37a72fa92594adc106d7ebaf3e1bbef410f37c59e6fb2f15256d41df6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.101.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "090e53fb319f2027c64c1fea9827d4451f9934313ff1051151f5d55f3a968b4e", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.101.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1565475d75682a0833008d4d85dcbdd6f2b56b1de4fc03bbc46192fc39e3a5e9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.101.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b7f8830b2fa0217536ca62d692821288bf8fb674e502d91d8ac7490c56c71ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.111.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e4de3219b4e33ab3dc3a414b00218ece844acb04f4c7425acb68156fdc4068d8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.111.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b5349e47de9df3bc17923cbd1f48c1594e5d43f4b5454128be0fef7ef66c03f4", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.111.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cadd8d05669cf3877def0c8a8e5662a9fed6700f974d72760f3c033d8c652598", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.111.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a8df2786687878afa52b9bedda622b5c831067c7809b37fbe15a55b64ed8d30b", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.111.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "70322ee057fb40a4a1e9b614b076f3aa4a8e1c3bb6dc2c00b1bc78e112931a31", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.111.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7034699e1fa0873f72cb3293b3b61dd8c25e11facae17a7cc3847c439948a0d3", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.136.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d0b116085a50a231ec69c63f0cd924e5b61f67033596e307367fa89edc6d3ff3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.136.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6d87dac16b4b8654e2782826e892f17b112d0edad5c78cee92f12c1cd7706da9", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.136.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e7ea0959428210a371446c97a7a6e976d4399c64769ccdb89875230172870377", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.136.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5df1a90b287a7a37c0e9b44b7247781ac49d053e3cd214622fca6bb0993c5f7a", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.136.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "73ea39fb95cb6c6c1e99605cfaaf36561c50221c188a0391ab5fcb3774c25633", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.136.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4182ad801183207075ddbc07934fb85684c327f39ae08ebb93d8379696a40a70", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.139.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e30e96fbb7e43de4aeadeecdc62c7cb10b335be7cfceb3d587af0286439e301b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.139.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e893f9fffc39aa5cabb12147222b5a5f578d30eeab447a61d3f99a43927e6e04", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.139.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a26c6713aa3b3e7db212207fc97ba7a4c09802acc15928e7ae919e60940d10b3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.139.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ef0245c118e67e4d80155ebd9c0b2cf89a613358492ff86314835fb9298348b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.139.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4fa02d8f26bb7666c8ce743f40694556584eec8457cc8bf08b744dd5a619bf4b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.139.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6fd767ac4ce998a460129f923a6184ff89250c7ce5d8a483dbefcfc2ffd2334f", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.140.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "76368fc1d5d0d8e8ed912abbf150b674870e249b7baa8a6302db30ae29003f9b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.140.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cd2bc65c10d8976661ab605f5471dc040394be506472fd5573ba153265402ccf", + "shape": [ + 32, + 16 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.140.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f460b0dabc9afcf5bb56cb69f093017355f25a4d2a3e771da2c17e549ad4a7e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.140.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ada1031f2163a7f28ef0a39b78f11bccfa934e4c4dc994a8f143733986dcdff", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.140.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f1a2dcb559f593d16db98ff4f7d42c62139007cef7cd94b4b59da2c7a5cbb755", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.140.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d76205e7e81436d1d8c361e1d435e93726fd3c95e6636023a74017c5dd80be5", + "shape": [ + 16, + 32 + ], + "shard": "model-00032-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.195.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4d89b432c993bcfb261dd8c23d81da0f151f0fb5d2ec7e40644f0ac8e772c8e8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.195.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b157889f126dadaddd6463c890877624f0372679d570443fe38aa46e6922b301", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.195.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ae36c7d7c1fea913911bb5fb7bec8dc7f767a3318348cea03034a9bedd15eb37", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.195.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cfd6cb31f53cc054569d9b3ad23fa79f2d4580abfcd6789eb43c63500225c68d", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.195.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2ecb1f29e37996ede0f1fc670bf31211fbf0b04e55f27d164fa91cedc92255c0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.195.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa77af6d1d20b85caefd14a50860ae7058dbafa6a050d06a204d3c38889137b5", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.24.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "623221ce54ed0934ddf3cc485379bc3268232197ceb0fd996d53ae574bac7c0d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.24.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c6c94b736502ff7e56cd2b118c2263bba61bba7ec7bf8761b470f52ad48d38b9", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.24.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c5249a4554cf00ece20eee888ba18aac73645aa46a97ba2370b0e60f129080e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.24.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d823f4af96165e39513b15d0fb261acaf0c953b4adf5cb81be33a9f2d9282836", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.24.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5870d30ac1c482ba78a683fff2b9986ee5ab0f142663c8d865c7f16f396e380", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.24.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b6973f64b5050f364c12b9b82289e7219b4c14710a1f19fdd2f7644957b4ef18", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.242.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d4706d4f4553d3ce3cbdc1dc5f9760661706da531df5bac3d68f7bbdf82ac7f0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.242.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ac68e1696ad35d94e7ed13bb538dc3d27ec4da57c6f37ea59edc504ce42692c7", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.242.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "db007c18d0637a3b4e1d1bdb07d6aefe38a9caf267315639c6057cf652ca6bc9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.242.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ed3abdafbf5e0b7fde97b7b6eef5b5f66e054e8f952edb2dc774e51c306c4a8", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.242.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "42ff7bf2a0ce5d3e03c4b3aa1d68901a79777717e37ec5020c501064312a9443", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.242.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9e52bbcdf0320018f5057c757657ff80d051e582d99831ef3ea739f7d5ec0dd2", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.279.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e6a6cf987aa4beb2d9ece770f968d113aedaade17460e35e9d5076f5d352a8ee", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.279.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "12380e290c494c19994a4c14e0fd8515892abcf27b5e35733922ca24d42aa3a2", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.279.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d995ea90c6fe2d5a808350910e71373abcf54291ec294238b7ec4f52d953d0fb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.279.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "226d99293acf1e6c966016449ce4fe4df34adc7747a0aaa1fe9e005e5f79c592", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.279.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "79b163c632c5123f65638117e5a2c4fe8a73b1ab0905778a09dc1bfc7b599684", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.279.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "37fa1c1c0d3a1359ffb6cea47eb9212e9014d31515ae854db305b57b151e014e", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.29.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31010e43ffddf6963c7e8d0704ca3ae48689366e837da1b2efdf800ba30f4352", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.29.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0912eba712622d60ddf9893b42aa338faeb5121c5a35a16b2a2bbc603034edd", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.29.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "677cbb805547aa0c6b83460b262db5e0d97b52a63ef01f4a40ea16db20749f22", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.29.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "884d08222e64ab6b0bd796e831597080b57487840273433a5fae85a0928b336a", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.29.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d03f5cf3f5d681984b151e654cf4f25c42f89fe2df3d87de0ad0771b68774f94", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.29.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec73691c394be0163b779c79e5b81259fdd7b19b6f56f7e62562adf82a30c816", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.36.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "db3c75316e6375bd9520fc8bbd04ebd9274bd5132d9718466508dca3b3d925fe", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.36.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b5dd6e8921dfd44459b5c40e29caaa0ffd23cba773b8abdbc04de676458cf53", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.36.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e516576cb82cebca7a9aaaa4dd035cd2582599990997c35a938b25670e5bfbcb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.36.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "37dc7a22f2317218344a3dedbffe0f308296a7fe7a26b09f0338f407bb38bbfd", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.36.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6cd17e1fcd66a2ee4231c0a5c07f3e3b1b4d4e192f7983d9ebfe6641ae66a42f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.36.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "56d2e5cb42e4e07b4736d59cf89022c70a21aa4d5ce31e085b8c55245d1180cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.64.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "512a4a0306399941e32fb1a9210a7b41c408fef882b1bb7929b33bcb0c280bf8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.64.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1961381303639e2fc215fbdb8c25b6920a58496236b011d3fa2c158072eb908b", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.64.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5f08ce4bc68310dd02a8c586d525e0c1b80cbf9807c03b0b81b588f5d8a0303c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.64.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "64744dcda27b0966ff0c7d68c7ecd80d6c9c3452d7ec40fa82459bd502ccf8f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.64.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d6b722d5809e1a2c0021d7dda605ba1dd64e6a2fad79d2afdbb4b56e6da2a53c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.64.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3dfdb276b78c1a62fab9070c0ed46a75050b153bef4328491f6cab7d0d5d962c", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.8.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "314b0b224c280fe9f29adc9ed3ec813612a583f58f8ad28b838a91e17c7c107a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.8.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0daa93fb92f30c3fe2bbcbff588373ee9f5aa63054f77a691b06c4f3fbcb4822", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.8.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9fb55748058d51cb52193f43946d2eda0a76ebee90b8eb278047add28d61d99c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.8.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4f38aa0739b247da9989442fd7883098b2f22a18d43da87514b02efe3445d3ea", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.8.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "173caee5c233a5488a3f8b673deadd347b33d158f2d3336e95dcc54b0c592a4a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.8.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e54e898bf9137bc873fbeaa97256c42a3eea4255cfdb73fb173e5370ff584d6b", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.93.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8f1902417eadc057112b9260d77d70ef9de75963f3b835e399d6ed326deaa592", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.93.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "81b2a992cf4d48e9112f90ab406622f7e22dd5bb9f79eaced529883754040b9f", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.93.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4f3beaeeeaa85994054ae54b034ca1fcac366514e7fbed5b5c37497b7a40f334", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.93.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aec8693bed273de7b7b8b68a034de92e285ec8acab429dcd7299454c0f2473e5", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.93.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "59c87090f12340cf5b0fbf2be14121d08d29fb7025ac2638c38d9052650f7167", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.93.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3f07dad79bcf0387937c206e07c7ead478f83bfe855a30060e619f7b79fadc88", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.97.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c45d61b979b6effd4e92fcae9ca4bdea32549f829a2c8b292076bf95122785c5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.97.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc39b73114dc595a1b981538ffb283115abf1fff989939c222e3bbce0c12caeb", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.97.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e0e94db925c82ecca910ccc87753782b008f8f60111408ed6bdb1998a0a01fc9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.97.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "708212e752b5fe001776d0affbf110f2264890140487091895b1e0cc55ec8a08", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.experts.97.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5c086bc21d2d1cc6c333a8d5e14597547073b27b3f339521c6e8b246caeb8b3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.30.mlp.experts.97.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0cce2d41515e37dd3c5a7e403e70c3d860d2e2aa659f352102cbf94eaa12d2c6", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.30.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "567aad29b2a6ecafeb484d60b1ca8b6d4d022ec5809da424813a3ca8525cb686", + "shape": [ + 288 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.30.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "c7dd623dc5faba7fd13558b0de462d1359aa027c50a7cb5dfe38086405b4207f", + "shape": [ + 288, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.31.mlp.experts.103.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3aa95944d6e07ff841197d345f3b49719d8e252c67484aa3041c483cc0191014", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.103.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "559d34afef67cf61da6cecc6fad33e6c57af6f2d0199947c6db83675066c223e", + "shape": [ + 32, + 16 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.103.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9a471710693f26e425d3b5affd6db97c6be4aad48c974705a29532b69171c677", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.103.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a27e5f3d28a3162178f317ef138ce4761fd8548c81e1ace921da322f25b39120", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.103.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d1e3bacae851d71fda2ac06fb89086b8c994bf61e7a94639c60d9aad97439881", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.103.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fea281030ca108592badedb6316ea92b5f7542905c819e09735a5b4598f37805", + "shape": [ + 16, + 32 + ], + "shard": "model-00033-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.11.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3398229320d10c4cc3af6ac49f1ceb7103fa97635f8b4f4b1ecf73cdb2b898c3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.11.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ca741841330380d2d69654f6e44d82d6c7d9db5244c65d8429ed827bd3dbf74a", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.11.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7df2c929e5942071586d0f76f7d112d7a27314a7351bcb41b242ee870fd6e328", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.11.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "606c8fd0783ddbdb95710116c88b0bbe9f24f3d85784eae09461384c9664d746", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.11.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f5d2d672fd90c7e0a1244d4cd04f8f52695dd5166e974ec32802eb5c65989f15", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.11.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5384a760cf5818c542e622189244d465cc24a308be499bf07e826a44c45df954", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.118.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ebdf6edc55d02d30235dc285f90ff0ca98fcbb25b3d2fb4b33b6c4277b6b73d3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.118.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "62c6e4dc1debfcd7e807f7df1195e34c66281a138b532dc40bcf7557dbc6519e", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.118.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7d2adb11f350c4a7b2a74e5dfab862bf43aa62a0cd1e6a9577269f829ec2a8f3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.118.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "988c57612b9d6d1c60716460de2a78efab474cf08cea9c51aeb105ba3239b959", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.118.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0d2e3c039afca0beef9a080cee26f85d72b2f2f8f260246774e4f96695720a54", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.118.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fb6e7a470cfdd3a0bf1895512ec1a6c749d18c6081c3e3f380543743f1173956", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.147.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "766c388c206773e94fe5df5d581c1e0901ac30d6f61c81ac6ad5f2d44b6b2187", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.147.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "496e1f044e309cd743a2850a847fb636bb48e95b4e3913cd0e9fcb853112959d", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.147.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9e0cd7fb2459333c61ec5219a61ef45dbbc3cbb5e8bc4ae074c3943ca62b4000", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.147.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14df6c5c7d0df1b75fc793e72ebdca705e190fbbc8057d1617bbf5a76b68b9e5", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.147.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84e9c568df9e9f0cebf220e1f41dd6b42af41ce9f1a0e2bf1a0f8d815a6a7b4c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.147.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a5f3b8c14c3f9f46adc02617fe888b0f394befac30094412cc1bc66b4810ff56", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.154.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fdc88b8ce7e5550ad0daa678db8f19bc9f00aa56f18a8b9186b4b08426d93b7f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.154.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "29ef5d1524a75d56be88aa5bf87a514bad38450ae0085d59d4de8daab0acca60", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.154.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d913a50270db137cf497fc46a874f36999b4ee7a98e8b8b62a51dd6e5a45dc32", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.154.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b3947d194501bf7c970a99645d11681ce1c9bb0b971ba7e43ac653081492211", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.154.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2204101240a2620505ecb0b6d6ff27c84a7bde8603e7153c4bd49cc24cba6034", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.154.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4e56e98b1601446880338665209ba9af6e0e4efb15686a091ee601031cc6d8f", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.157.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "681fadda1f4d95d9682c42204245fb6924c77f67181f09a5294c3e545dcc869d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.157.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a75f40fbc9275aaebaabca68090b86aefeec1771ade2cbdba43603925e07b55f", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.157.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1d64f3a8475d4b5183ad3212a4085740a9af3cf0738f74abbdcf77696d4709c0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.157.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b7ba59336f8cf1db4c9bd455f79f1e9d5ade52ac509998f1aa74f3c58b016c22", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.157.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "21a084715cee116ac26d405090405626ac1f9ce97422e56d140e193c537f7fe4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.157.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4d1fbd3d99507a6cd333e90deb5ebd7c47e7cb17273f10c797e1d96aa7ca0c36", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "22481d4311e532251eef798fa40debcc0cbc77dbdbb80178f6a71ca2f46ffe01", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c72682d7b24efbe8aba24a70ad287c5e1df8231a6fcd3b9716681f2d0e86a87c", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e04da501cde67bb88fc5199f765f77f282069d16a9f09ca95187661f94b7801d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "82f6cfeeb1e65c1d8b948ed2afd050047bd07e76634c7066425ffe2f238ad588", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "80edc3354e5679ea08852772ff20782620882e31ba333a509b587882282f1556", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "46149889efa0d5a9887a9a8b24ed7ffc74def655050293db8b94bc1c08b58235", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.197.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "027b4a3001bc3111be04891888df057519e028bfbe7c079c1326d13aa625b6be", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.197.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "55b4f9d2bd584a7a1797aebd84954f416f4598f50c922e7abb8e29d6812bf84a", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.197.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ca5a6a799415ac255cf1a5671f3d3eee371167503389214defe30cd377dcd45a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.197.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e70b829102f58e42b6273f2e12189d522cf6a47682e8646fdf66a5381da9f1b5", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.197.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0f3f3bcc75d012e163a206fcf94536f4440b333b115dcbd4a0845a49d033fd8d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.197.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ab8fd1ecf4e4869d7fd9464280bba4c3f0a7b755e2e941d9ad5ac13a924d5df", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.210.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "22722c9d5786a237f17f770df77c3cb6ba75f33f7d5ef7fd590315e4ba6a07d2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.210.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4b1145a4946112dceb7469cef5a7e917b93bdcc736e353b7fb12cd2040bc51cd", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.210.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e701b261adadd9bd83b37d1f9ea3fac70c27427d94aa4bda3c22595834ae2474", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.210.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "689138b0be992103fc135ccef40372c749983739851552389e49247ceeb6845c", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.210.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5744f9316c7a60eda623c0c1025f46876b07ad028cf7cf08cd6c12876b393f40", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.210.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38ec87924ec00f873138ffb4905b26baa0a05e812aa1cb25eff02434340300b2", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.241.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9d1ea3cd606ac703480015e7fa68ae9673288a583e29a943ebafae73b0f9331c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.241.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "340caf9ef7e4c7846ae53d92bd06d15b0d42eba8a528bca844a57d40b25cea0a", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.241.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eb3bbed3017ebde672bf1a25104ef10a34f17586bbc92b37fc5bfcd8445de9d7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.241.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de306e2862a9c1ccd04f81d92e0e86784b4f0bba9884ed364251108b56279174", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.241.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "085b56538b67775a19002d802e5fc1fcdb28c8ffe371e91b6825bafe760a3d53", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.241.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0a9dd584d664d6ccdc89bce235b49e458b0628ab0778b3c84554053f878bbd46", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.248.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "153fe65f97658b9de765e378c35774573eb07343693b6d3dd9cf5c9065120c48", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.248.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dcfaacb9eb622a3ec8805a0a07689a22d09ec25f0fe9d4ee851d8b12d1f12092", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.248.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "06292b0e827287f5cfc2eeb2b0b492dbd1d9caac82c25e9a28b1e6dbf38fc0c7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.248.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4c737ba0481711d0dad8113df02912d136d4081ff739a7fc45d25dd47b005e1", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.248.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a4c607c598755e63b8bc7d0f6d9cee76610a44f036f93d5a34e85ff34ba26661", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.248.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "25aed0b18342d857910418687241d100660ab1f24eece3a4f6f95eb7e93033a6", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.25.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cfa093f524c1ca8f71c7e343dcdc3cce374222be3a6e1e0e0a881d2e616cc4be", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.25.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b95d857d3fe3e75f9499344e2542b0ff3f35c27b27c77911274bff4ae808d84", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.25.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7ab984b44820978a0e58ed40765bdc74eaa1eb066b7ed75c4279072df3f39684", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.25.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9eb77e0f9bf7a0e76a924189d721b4f64024f52f6ffd9ac312d7e6daefa9cc68", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.25.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8882acc2112f3a884c4ab259ce96554469876eb13cd53ee2745a26c858554f21", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.25.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "547933efe3a9a43198b1195d5a3bee21634969502d19ee780e0bb92f29d981f5", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.251.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1a1d1966622157df1829cec1f47ec3e9b9c8ffc9d67db0da773a4e5771471456", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.251.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "54df32216f757fbef3c7104527bfe3ee6b74b46f205729244dc175ec13e8de9b", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.251.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "515891dd53f7f11826569714c66fba1a421e8116b5e2fdd5765377c398a29e3d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.251.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6f44d13b99c6ea9cd7a9698b4d2b84330f4a735924f1c24764caa0d6b5edb92e", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.251.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "817d89de4c1d87686a27fa25493d39ff9f4e4dc59919f920bce2292492d697cb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.251.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1f2ba269031f7dc9b578ed919ef7c79cb66e79e8f57dda015c25bd827f8ffca5", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.268.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7aa3607f3ede79b46cf7956b70193b1d6f4ce16a6c372c067a9cc20580a514c2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.268.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "95d553b6e7e1ac8e191cf0280e0e7ca5f6d313bf0f9f6114c7c38647c1cac4af", + "shape": [ + 32, + 16 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.268.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "29aa0551224a4c18b30e55ee0f6d9fb4a0413ea0b7cfdd78c2a222ffe41abd92", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.268.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "43d1c6ed72d988c0853e58fa85115997602adf3e77f29bf9e4982feef93ba01a", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.268.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fd9dd05091d619818fc51d91a251828a3bc234c2bc5c330b70794b6cabf70b90", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.268.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c5bd118fa48e9f904c31eab227e5101e20ff0ff3647fcef97a1d2ca95fed340", + "shape": [ + 16, + 32 + ], + "shard": "model-00034-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.66.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "15f247fe488bf2c54fc9faf7db9cc246e4ebdcc1c8a35232b88c1537836f01bb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.66.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "63b026270ccfe7a14bbe7e593f556b46a6202f0452cf88b8f31f5fcddd27e7ff", + "shape": [ + 32, + 16 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.66.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "998c95b1022bd81962531ccec07f60139d349c734a06c9659b6ffbfdaad87ace", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.66.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3fdf25b8864a3c89b7af5ce670d494cc8eacc8f3cfb43d14387677e5f9af8568", + "shape": [ + 16, + 32 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.66.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c470c9086ef5eef4bf267c102a10398b374cef467d614de9b268154fc5a913a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.66.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fda956558ca9262157d46f1e85f3d57a6b804b10daf8e6bceca20d8f4df3a4b4", + "shape": [ + 16, + 32 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.98.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c1be2e9704c6b4e842426aa46cf641b1ae0747a41ab16b1046b844351a873f44", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.98.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24cfabe1c7d2a4aa59517d71c1432939b163ace740e72afbb017e070e177f630", + "shape": [ + 32, + 16 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.98.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05d27216e1a2f12a7c6301c698f2befb85adb507269b8b33443a541bff70ecd6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.98.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "11b21417cc37af7ae6f53698e785ab488612b87225a2130bfb63416660d28f3b", + "shape": [ + 16, + 32 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.experts.98.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "02fb5781edf6757f6cfe18ec6605e066d15d3482eec77bd0381acda096947fd5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.31.mlp.experts.98.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "83b6488f2ebb4bb7d1851c9c5f9baddee9a724c3fbdb11909c6bcbfa70245ea8", + "shape": [ + 16, + 32 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.31.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "30fb715ccbc17a155143a1efe8c42fb02e205164e325e12cfaa503f3e199796d", + "shape": [ + 288 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.31.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "e8a3f99b052f2ab4bdf970e279987ce2f7b1a373b48ec13574f1c7b290ff2bc2", + "shape": [ + 288, + 4096 + ], + "shard": "model-00035-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.33.mlp.experts.128.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "16717b462e80b5bc7edd3b7d064003c5ff3dd3660df0c30cd28dfb8fc9d1981a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.128.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "74fa5004ff61f249ea13100f52c9159a407aed6714fb32820b70795724600a6f", + "shape": [ + 32, + 16 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.128.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7efcbea592d639cb1f706146b72e6bbbad4b0c23701284c8486d75b6d2b5aeaf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.128.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "283cdbbb598cbf088bfeaaa1b24fb92859a4d081a60c00f59e6ebc1e9447582e", + "shape": [ + 16, + 32 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.128.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2d5cec87c9510b34dbbf08bdbed1c4e996b978d55923c60b53150dc0052914bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.128.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f2c978e5df164502b708855dcfb6cecedcc0fa40767bee0352c375c7d7a5a64b", + "shape": [ + 16, + 32 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.130.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "81f63b4bfb47141c9b06ce6ca113253ea07f63f77acc9e4b01643d340fa48e6c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.130.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d034201cbec950e7ce265031e968800c47b7c213c604e54ddb4126e9fdaf9535", + "shape": [ + 32, + 16 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.130.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "41c3aee426d2014ce35e4c8f10342b37b6a2cb62972c6012bb18eb321711d8de", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.130.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8d247d9546de14fdd5052d7d6fbdad1c0cf27f9077b8d46802710a13ab950bb7", + "shape": [ + 16, + 32 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.130.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77bc9bcb671b085359e1a3c22d04b932ad957783483caf7ea2cef6bebcf6002d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.130.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fc21c2eefdf57330cc849324c9cc4215d803e17df44503bc75309cd58763be5c", + "shape": [ + 16, + 32 + ], + "shard": "model-00036-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.168.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "428b87265a7525e215274b96e8d980115e238f3f57e16e2b777f8cf6bdd31e48", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.168.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b505c092e9eeba7c6d09dddfd32ce18893a448fe9dbafbf7c7573b59fd304bd7", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.168.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "804e888be91fefdd0364d69a5ab9acde53fcb13d5183f6424fd748a886e91e0e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.168.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cee5bd71461cf463036d5156fc7e4cd132f710e1f04ffc5f4dedd3635a187eeb", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.168.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2bc5a077a57e04fb6f738a871229cc67254deaa94d93948d9bcfaca2f4035ef9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.168.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de3daa5695822ae5fc06f2c0bc19dec5e57753579a896519d8a145970d851066", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.170.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6b10a48a1e276da97d12681f5ed2790108ea82ff8b438822faf8018213a51ea2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.170.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0e5f390f60450379ee8b4294f8e9a1cd5583f386d20c98b347ea87d7405b55f0", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.170.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "210aa00ed96a25368234efca3bb0e6dc17695e293b560b2ee24dd3b04c7a0782", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.170.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f28bdde539120175c52789e1e840ee57c87dde9a7b9d25a68cb9c5109bf297d0", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.170.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef3c690eeb59cd697248a4fcf49ff2e5ffde5b8e9fe11b4a469566145ecd5508", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.170.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0054f57a1e096a5ce56f7c03513e61645d05171e7e3a5d3278087abeea97558", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.171.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6c3820191f371f4a294ae5739e5b0179118e3ec636177eb8ae10e6ec640024ca", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.171.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "04e1710809dea02386d807af2f83900f3d1465726869d87fdb0f4d6733f33186", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.171.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e14a89157bf0e704baae85c807e394d4f78e75c5e33113479d9912fc8c63a70c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.171.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ea4af9d62c9e8b6ce2b6e23fc15b0d5918189c0d775d9e7882f76278f0b7899f", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.171.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "70969cd0cb5319527534ea4dac0ce6d1cb8d60868cf1e797d07d17038b7eae9a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.171.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3712bae83d745886525e1d05f782d319797d7648ff1feca075b27bc61fcc4da6", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.172.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "59fb23157917bd2ace712646f071441809c5fba1a10617a8a30e4a75cf3402e8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.172.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2de8ea8dfe84c8ff4aa7d3d65ceeaf8e5b29e5d081a2e5e6b519ee526a93be1f", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.172.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bb54f3712c924ee0713f3e6dc565aed0ae13bd84a745e734fe95255caefc778c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.172.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c0fd4ee2ff1463737506a10c1a0d4cbad9a50e47945d6df42b9f628e47ca1f6c", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.172.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2c377b2dbbecf75e52f09220dda730f4e4a650a8e185f68b4b8ad4519226a8b8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.172.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f62782138059d661d547f7017c015e06bdfa7c323496175ccebe0a5915e4d1ef", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.202.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6731ddd0730cc2c6d490d3e67a7951e6360f5535424328c5a7c29de3397a2404", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.202.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f9b385780fc7c79e31e25e40b42caa20c9d95fc78408518bfb6777f9c1c5901b", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.202.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b705cde78c9464fb17a2c80545f87d406a9da7bbefc939816d036882e6941a05", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.202.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c9b8fb7f1a27770879365f6e16a67ed418477c0c09157c7a7a460eeb4ff6baf3", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.202.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1337bf30580dd3f76b0826c186f320a0d13e3f615404b707bdb570ffab64792d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.202.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3621d575b691b73f26eef81a890595f2d6317aed40c56853dd8550e0675fb661", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.206.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c4d40e67b1381340b29e20a48c4643be3e06ebeeeead9cc78151126337c7ae9b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.206.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2b4928634ee766d153e04a55ce025725a528c0e28d388c6c99c2db7ecd90c217", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.206.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "993698a1a73169f29c07a35ef86ff92ff8c6d13c59b349df6e17b3127a4e4e27", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.206.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "444f2175f31770322a25113d8fe630177be8b2ac8fc4285b0f7749e632779b47", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.206.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18f15d7bc7b6b56b1ff85e2e03ba418e992bb56968350aa949ede5a7a68ab6cc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.206.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "502ce8d3a50f66d1da9ed2171fae0d84f7fa82b92a83fe79eac5a1f8564cbe5f", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.222.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e603ebca35a5ef92ac8920a07872b730cd198b57ad8eb4a7aa43e904f55dad85", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.222.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e8897a75fa9c2d7d1d98ea95e911e9bd4a0aefe2011f6ba12916236c54b54ed1", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.222.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8d178c340bd2a405e41a99743a90433dd4e2031e7bc683e1e37c0d59216fd8a4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.222.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e01e869df66f8b9127c134c60b1d6205c7afb59fcddbaeab6360c208ab2c266a", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.222.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3919a82a983491a4ce2519c3c9239438fb68974da7e19cdcb586cdce2c65e9a2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.222.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a5d9471b7737906b6453b71e0a594a661b797bc43fbf534da06e473840e5f724", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.226.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "314c0c7afcb6eff3e434c1f9f9329c675ebe32bee914afe698b4e1b665b8c81b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.226.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "485d3bcc798d663af864b6502b4b5db1a496fcb432a9c3f6463a0f736db0e89b", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.226.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "88610271beb984345cac54a2bcd46d7b33758868bd1e5e9182eb030b7660765e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.226.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9f6c8838dbcaea0e99930476a8ed4a5a2d568a3781372783955d4b1e275a0a0d", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.226.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "34f2381ee466d854dfaf3022cfcf910b82205dbf21fd6e95f522f648ee9eef43", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.226.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8cd4806f965c01c88bdfb2733075e5a038741840b6cec2d53bdef1985913dae8", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.237.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5e5d1ebb72a5866f227f98b5ddb3d4a007c3b16f90beda7bc66bce4200bdd39", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.237.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a82beebbf7137cb8c1476b36a8b6d3264cb4d7a2154d56f09e3ac337cfb84d1b", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.237.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31a67f1ce5f291afb835b97385d252d578e2e4e8d950cfb3eeb5dbe792ff137d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.237.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "31ad200e47901641c7c9143baedb525d9c594c7aff0f4db497d23d7eba8782cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.237.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d557fa5da3bdbf6a78df6aa5849ca3c485044ff8855b3491ecd5e1a78047a89f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.237.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ad5a1a25a93ccaaf0b81a6299a60255a300d25490afe36c810e723f86e0fd2ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.252.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "456e8867c2bdefa5451ff317ea08207019b5f53cc51f0b584c3e26b6bfba601d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.252.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ff535654a96e809c375c420b1f00059e585054612c4a2b9d0e10d266bfe8a0ae", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.252.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b294af96163b256fffe250c5f3c1ce51136364bc63be45cee264b5344facb4d6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.252.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4f3321931e2ac5359a981c6d550fa9779dae700b3b444b7b94c541fdc7226857", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.252.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5e4fc4d5cfd22d8e68ef9153e8e553542dc5feb5b39a205252bd40a29dd0c8ac", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.252.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5ce831cf357e94343b0d96885dd0b49d8a8bd223b3cdffa96076118faccc41e5", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.256.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e2502fd21c9bcad8c18db54f5dfadaff6c18f22ebbe85532edccce8e24d00943", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.256.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3eecfa4fdc353c4c8636a2fe5675c705701241a98cf34a8cec5037d4c73ee1d9", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.256.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d6a142602aa9505ba3819dc773cd08ba44e4284a1f1d841abccd5dd694a906d2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.256.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "207406e19d4eb7bdcd33c15c6bc4c224879765072005413a3213d659e4f0343b", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.256.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2426f211170b6f9fa1715236d16bb9503905e34c1694e58156212a61f496c695", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.256.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec49697ba1e51cb4e214d53cca723ab5b57fbe09b67422b2bfa4e6b9b9954a89", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.61.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2ae552eaadcfc7aa8bc9c7ec27fcce1ee2a274fe9eb89ed6de13091dc2e52548", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.61.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a803358b9243bcd5cb2e1c213aeb4fff54fef0a46a54377b11709e7f54a6de76", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.61.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e9a29880bbcefb6ad414197522d65b3d4033d71fefd260982730e3d1ff713349", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.61.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b58a349c7544fa808ea06b469a11ab74f97a183b186c382826a3fe5494e57421", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.61.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "137581e5e7486dc5ff2a80e06ff60c48eec1f3bc1a8abd9167a8d50399bce3e8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.61.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8f43465e43081da061f6af313790a531ca5e4eb18af8af006bd09233c41c4d89", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.72.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "67292344fa494b1eeead55c2626e9b035b804e8fbfbe5f9a85ad38e457e23bc3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.72.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9dd080fca7d770055ea909501c44c96a9a8242a54674b843ef2142060b086d96", + "shape": [ + 32, + 16 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.72.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ee4c7c5fd06a18715c072217324bb014fd01622bc0a3d06fa5a944370941ddc3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.72.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b15335325ef0e47da5186f37373064c1739d07cd07ed052b74f392c871cd2466", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.72.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f0ad38c06335d6da51e49e9a9764914fcb59883a32de03f71d86e647c5efcaff", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.72.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6f04600cb19ccd09e7c496f68166d18939beca66102f963f322ba6d9c39cb649", + "shape": [ + 16, + 32 + ], + "shard": "model-00037-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.8.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "49685d98345984a8fabef405531b98a1ac11c6c61dc7df53a304dddc6ee8e580", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.8.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e2e7bb34511be603a80c0da508fe4e6fb2ea40604b8def0f91f6926e52e27c03", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.8.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "081f1c732e857332b2d602a01bde8016e42cdcb1eeb51ca4f6cb10fd5012ba6c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.8.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8d44c9c4018b690e5689dd04422720bb7cdd10756e028c5ff1d92046a2e2cc46", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.experts.8.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "243beb9c1dc69cfb58bb46e516356b1f7c32ed04d77ee80df3d06ca10184785f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.33.mlp.experts.8.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "20d143f29dcfae5b7698ad1562ac2b0c1d2efe382685e26f1e105e5bbb2b205d", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.33.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "4162d3dfbe1982ae25d1ac3d240c1ec5ed3ada932a447ef98cad7e90d8ddc22e", + "shape": [ + 288 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.33.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "b346611e94744938cf08069b3f0f93b1097c7898908fa9991f43aca46301afa9", + "shape": [ + 288, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.34.mlp.experts.137.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "87ecd9d95bdf28b2b3031be58e4c430761596e53d44261965629a729983563ca", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.137.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f9eec498845946b3df6656ad71ec875741f2b07239c49a65d8a90ac90e4a4f78", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.137.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "70de994b4d14cb7fa81a3efaf15ea0c06a2ef6d874bde19be066e7cbac844ec4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.137.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "71af16d3d7385fc934eb956c3ea0f1f8c234e9d2bf0e697f03edf917c7b7d9c3", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.137.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "47bd3b28d47f87525f7096219bb11e65836c165db4fabf1522841f4881f35d08", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.137.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "705663e76c889c99dd4b02614e3aa50986805559d4f5ff4c5aeec181748b4eed", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.154.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5acc946429d68bcc3922c51298c431b6f99fb84de90862f34a8e80900753bd61", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.154.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3177ccc8cb7e8073725538192185f1337e8dd5cd8555609786ea26c9e7c548e2", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.154.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4720136c3e06184f784faa9accb561a1ac8e44de0ccbdc1b1444a58025c3ca0f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.154.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "23cbf83cf935cc0d8651d3153268060a43a8effa627a9a17a14bea3a0a0bf54e", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.154.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a39f12114eace2be459a274557a310a7705792c76a6620700a792bb058e4fbcf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.154.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d6325bef25077f8177a59ad56fcd72076801d33f747ee4ad6e9549a9d05e3a44", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.172.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2cabf5e87311b558d586d9ce045d3591e12377c7c52538655ec26145445d6f05", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.172.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16f4ccc08c6f3a4578844619b5eea9a6d24b9691cbe51002473f9fe3ff42eee1", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.172.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3743338a6374918955537d71ebdf9387243a2e910c04a7c9d47870244c0e5ce1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.172.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c40766e21ca4ff33545c74bff59fe372bc33eb07d555d31e323b4f69402826c3", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.172.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1f943b35833f8603d89fbe6dd1d0e2a2079cb150106fb7cc072d76a871be1a7a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.172.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fe0cc1a27c292935c00f0556ff68bbdd7270c34b27c15c5894af1534f4acf861", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.18.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aabaec670bbf4209c9dfda743c7a7769f9af86ec2b46fedc9fcb75383f3c8102", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.18.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f1cfb00c197615ae9317f3fb765429e4e79412d9c937c2059dd428f3b8727402", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.18.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b8c8cd4a32869763df88354eb17c91a31784d76561559dba79abab355ea2983a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.18.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cae120efcd904ce201d47c54064ddc17b4cd395e807fe0ad993962cfdc440786", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.18.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "98cea1ec6c6fc416ab100d96f453ec7379afedbf05e35f755c654452c0e9070b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.18.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e37a16e6cf189c49687b19adb508bac835be59527cbf3aa85a3815c37919c7b", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.19.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "824934369f37cc14594e9ab5003223e966ef0cb7fb412532b975c4f81d8b6771", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.19.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7daa3b82657900d509335bfe9004bad883ef0a56bd76604bbffa3e0ca1a10716", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.19.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a07baa34f8f444dab84ed312b8d015d65a4dee3d04f89c73bf5fca54c9de29df", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.19.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a66ba04b46c7142c2f383d89e103114a76e0f8c421f61b8d59ee9c33533975c1", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.19.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e99b66ea70d6703a971094bf082fb80b00fd83f684f1341c98e7501b6585cf1b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.19.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "73ad57b5ebe2915db70d2c85e34131db821e516aa0b184ec132367110469210a", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.197.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df5c3d23d96eaf4aecf81bcd1d8ace6a57a6ff4c247f62f2dddc5bf34f2de239", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.197.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0f2890e0798d016a9571ad345ee1f775f4007b66e93866358f70eecb85fc5f7b", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.197.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9af02d73c740b1c234908c702d0566d740f527eff57fc23da6c7707f24d779f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.197.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2f31a8e65329f2286383b098df5484a27c8ac7bbbf71e7ad76af8fdf217006ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.197.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77bfc5ac2a447968529aadf22b9af333ac5ae42eaf920769c5a9cdcb14810083", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.197.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "68d247f447d12d1f5ad3a6b1c1656bcecf1f33a9d5891c789d5cac8e84af74ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.255.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a1daf2e4f8c03a4d8af7d40967c81938a7353322ee2f7a3a31a7ab3e887c5275", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.255.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9a2387cbb0c55cb2d7816c7af089260eb8c329b8faf81fda092810d5ffb5c577", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.255.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "76213bd2b43c9c326b70c80bb310d1af45de30f02dce9596eaf11328ec204993", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.255.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6dbf4a0ef7d102e7eab93c303c8227f631fab34e35a9e5476991efd032425f27", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.255.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f32d4081908fa99551fe92bdab3ab65b5545f7d9513895a8c873861df32d049d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.255.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e4245235f2524f199bd97c4ce77de0dff482b2792c29d99a5cd10a8bb5659796", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.258.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b028d2b015f24545ae2a97e8213fc470c38cbb62e0afb97d81d23f8d61c939dd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.258.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9a06481d354b773558fc089a0556bbd97fcd7ab0384ba8beffac68dfbc479a70", + "shape": [ + 32, + 16 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.258.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bc9f3d2a9d8fadc68936996d355916875697135c6206ee0a4bec0b44fe937fb6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.258.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d5517b343a8ca414f4079d1e34ceb0f50cca0d37faa62ca13ccacee46a8a5082", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.258.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0968a6a144fe558cfd72e232991f4b71326be2a67ec177e800ce1c1bf2df82e2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.258.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "76187e28cbb7bf3e36d002a4d3686d75628c0c28d9fac6436470f66e70de4e3c", + "shape": [ + 16, + 32 + ], + "shard": "model-00038-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.35.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b6b40747145690f8b8c499f037bfa1ac994a911fd49b222b8f57e34985c2e44d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.35.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "80ab89c76a4380f29dcf77503ef4e4a6c181d34f68a9cb75198311fb6d799a83", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.35.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f4b0b45a6e15026e01926868f2771f815bbfe0bd9d9986af82ea893998b8b1da", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.35.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7b7b6d086c728f7994feb454d7faf6280e5416081b7ee5855962f0d18c860f29", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.35.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40344100147c7b33a6cc6a2cd53ca9e5c54082591bd2b37316baaa72d518b48e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.35.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d14bc9a266190adc96120dbcc687487baa9fe40d79ae777b6237a5ed7b97c214", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.38.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e058ef73483eda15e641be96269abaf09684b992d74d4e1c6474a6a285573ee6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.38.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fb9119d28ac935ce89c53452c570e84c9c6ad65caaaf4b2d48b8d15145e9695d", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.38.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "58ab6b9971a2330031d5ec8b88b4ed487d58dbc1458a4622b5bf72a89d094d33", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.38.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ed004f7fec823e5b4d2d6cd45cf6591463a325b9696dca05640c3db29267496", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.38.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8adbdfe33b3dfbab63c5e8c790ffe82f37b1c66a2e7293ce73558137c2804ae", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.38.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "524561f801a4d0739bfb6fecd2a313f7ac994f47561b0e6bf510962bde16e5a9", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.40.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "54aa74c29d24edfbb67e71c566dadc650ddfaf121100dcbd664b3878531ab3bb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.40.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ccce7546fc57d2948f3af57f19adfbb176a611ba262a3dae6605b3b7fff2ba72", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.40.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c48bc59029b38bc6ea806e675afa731c13014c9bed3846e21e0e91ba2844797", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.40.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e54fecf0180e2873b1005cdc454f759804160a995b4902a2687ae5db1a96ef2f", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.40.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ff296e14506d364721d77a0b99e4b6b409c52f9d9c1ab1c1e60022303ac2365a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.40.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bcd95c29f254d1cbf2d58440dc08d38b54e737501ab47d1d9df26ab19f48b0d7", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.52.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "81c994c01656838756919dd20398eca693eda403165978b3ea0f35ad18df3215", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.52.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "429a49113e365d507b200397623a707972b1eb7ecd74e4e8e73931f13778fa29", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.52.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bb165c04b1c3a35f04e77b4531e3eaaf57fbf36368ea311638f8627ef6b8af4f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.52.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0febc277194e8110e3abd803a1712be97ddfb41b8610bfa69a5b70d0e577b3f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.52.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a67d189d095e8898d329a6a66362220a87bd0e1df6949a09e655f42321502e31", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.52.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fb7897692b7f480a3f4792b489cf31b8d410610504f0fecfa7e6fb6757a1a9f5", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.59.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7463f0d85fb93b86766fe200dce44b5fa922eae89e7ecb4e24bd73e369cdd008", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.59.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0299d7518668629d2486632292025face9691e4618021a3522225b0f447f5b9", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.59.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "147bfaee59e00848579ecdf8b6232bcf13d01d2ca64573bfb551efbaddea4b68", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.59.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df1c9038350c9aeede1d32a4a0f3fd5189497fd82de9a6b9bffa573ce0c2c982", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.59.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6026c0219c60cb1a3f43260a1c0f668e8cb839e224f6ce843d93d86d4d774642", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.59.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0170cb6d3e9b8a1491103fdfa143d40ca610e76d252b5775cb234f39900fb795", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.80.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1b0e4b1f67f53db361c36bff0283ab5817ef23fd14fd20c39dbf354139438642", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.80.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "59dd9b20ba6de8393b54c65cc1a7dd9128bfc1816b8a3e41ad1ee17172c86869", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.80.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0dcb17069e48a9a2bb816592c54676c668de38a3d4cafa44458dcbe996467340", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.80.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "67b327b0a1b21d0a9c87a9017933764c58a76d6299aa5e3d9519253bf6b61deb", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.80.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ce8c0ebccb09da0638d25faac0b648d85e0802406e5e02f5f85cd69ccc01ca8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.80.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8177ad1a72f44b57830b891eb9e2061d33022494e95b0072f9034a64b989a865", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.84.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7a01dad56b35991f254703d8fa1e1369f664b2c4772893392bb757c1a7a8a75f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.84.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c9033ff61b71f1b984eaa1bd003e95a37f07403ed3e4565569acbd1527468616", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.84.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9b99abcd52443a0153855483ecffd473957a98761a8f43d33e32e93f69cbd02c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.84.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "954972dc22f827906d97f3d1824d8edeeef5996197a21ff7c9e485675b5dce00", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.84.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "28a796004e2a9f263c254fbefe6a4953094fd13f9c77cba5737a363b7f54fb10", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.84.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "854784797526c6a52701d684a25bd7d8d9fa8b027bd690138f09b9836cc44936", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0ef17b655fd59f87f0357f553c91f9a35c0d76ee995a25ccf21ab841767a016d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fbe5817d4c19855d67a1a5ebec5eccf34443e5bcb987ace9436d1e9bb17ec1a5", + "shape": [ + 32, + 16 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bef8eb1c4083352b5bac4b331b524f9b85bf495d4c924f85f7504582a956c3ac", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7485b656bc12668a8ba3b620207972b7eafc82e4bece4e136b07a6542bd0c8c2", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "391b37f256a1393055e93d60b7aa9da242e596d39e1f41c7c19bffe74d1faf45", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.34.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "30ee8aa1c1aad65d46cb7391ac38852b244edae75778fcbcbdfb9123d3de2306", + "shape": [ + 16, + 32 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.34.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "b61ddf7026ec518e4bb615dab192258f97c0c07f3541d5b3b08f82180bea6d9b", + "shape": [ + 288 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.34.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "550bba2ce6697094f7f2fb5a01fbae187b670cdde3a601f9ff23d51fe01f0b49", + "shape": [ + 288, + 4096 + ], + "shard": "model-00039-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.36.mlp.experts.10.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "529c1e5b34707e61613eacdd0872f0394e2ec2ec16f9432d0a7dbd29c47f48c2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.10.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14b9bf6cf9df4fe2a966c531ee8d632ae4ff79b83e66182519925313159c7508", + "shape": [ + 32, + 16 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.10.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d98edbdfde57d40a9684e97f8e79f7ac97f427c1b1e70aa3dd56667fe594e588", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.10.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e2db8b3d0404276ef57aac04d41e2934fb469896a6347825eb53289a50337047", + "shape": [ + 16, + 32 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.10.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1bdf0dc99f5a0fe2f65ff23e526bf68819bd58380ce36302669d63e12c0f6174", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.10.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "72e2fac06d54dd55262555f051604bfe7fe67f3d383432720f2d522ba6c6e8ca", + "shape": [ + 16, + 32 + ], + "shard": "model-00040-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.106.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40428cb1b547589d8aa8c303eb650b547d1360c4cc373a14ce853447b5a55f05", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.106.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "479c00795bf3c16c7329f277b620835e86fcbc1c1ec3c9510fba776798387a10", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.106.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "90cb22ef74888f56e7d71c3b6a0d8f76781f1fcb7a2f15b2c7a06a487e46c758", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.106.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cf14fe9bd9887cb708dd7c1953bb96c8a779a1f3aa0c03a617fced43a382e397", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.106.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c52f91907f188ed3f0f8a415aaafc0d83c03cab19c2691fce0f9c6ee11dc40f5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.106.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "478685f356c4577c726c7868ba8b9b025d4b53a0c2c823e89536614724b89e74", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.147.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c8ba4f1ab54c5484fef6c810391aa04c04426585f82dbb1dbee4d9ac6af2b3a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.147.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a40fe24a5c61ed29937699bb46c732d3ee4dfd1f613838357cdc0a6809685aab", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.147.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "55911f072af8b830628c92c131dbe72fe5c07a0da86e4312fb2f7631e1867517", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.147.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae2593600ef49400e3deb537887944d98d75f805dc920c14544c944ab252f1c2", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.147.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3339eb69177929693a2078f0804651b2d6e5e4e52a7c0d65462de1aecf0a5a13", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.147.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bef08c896e8870fc8359747d71f35c6d2d01a0c8b29ef5218f3125aa35877b99", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f31911c4201d6a5fcc5534c847acf02e92fd85bd7c20caedb7c2df4b2fc3f827", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2dd4b97c5c3c6043b37bc1562cf74b7cc24b8cf05e1d36aac772b74cba5ac36e", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17d10dbc4b5ff2f2cb337f2bdb27338925d34dab30830c73f56728ba1586e79c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "af8a6c1c97071d70c612dc71332bdb63694ead3571df069fa1daea80dae87429", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5d57f2dd7a47745191a21ac0e34f8079e3e9ee44c959091f3c49bd650ae7a7ed", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "46d8a4805168e32e0a3d7b6aadacacee1f7c55e8121a43d519b8639229ee1f36", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.186.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5fb5f967716134fb87d8e7df681a5cc35b8cf1cf2b8f64cbfcc41be4b7f579f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.186.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01e06878aecff39672c4edc8b32e39b7ddaab4d511c15e5a49a61dc3d32332c3", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.186.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1274f56ff61009f9433ac9fcd844949e497bb6b2c4649873377f925a3c5f61d4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.186.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3d397deaa754f45751928b814efd0344ac5d74c674eefca8e869970581b244db", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.186.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4651e94ed4a09255db91cc0d4922677d2fa8d7476ba7bf4680009e862676d7a6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.186.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2eab3200eb8b4fc98cd47248d66a5c2f02501cd91901801d768d5da4a953aaa4", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.187.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4ced2c5a7bb12d6b0db411800a01721eb1c1582a263ba9b3167368ba12125aaa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.187.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "78620009bd0b10d0364d18914b1708f66536560db668c9d42de8697c967c397d", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.187.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e22d54222adc5090a4a575fd813761bcd7ec5b27bfeea8d7b8ee6cacb43089e0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.187.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec9b798a9ebc495cdaba85f4972a203bca77735cfc1bc03ebca29f6625b66ee5", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.187.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "831700bf6938d09db76197cb534bb478a9db596303c6b624ac598c0fdaca91ae", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.187.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d3d8b1b9795dfe568ac1e3917f21cb8c5b67eefaeea3b01b020f24b7637540ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.199.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "feea3b444641345970068884c23061baf108be9705820353a07f976191612e68", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.199.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5e8608a76f74928bde445ee93fbee5d7a01e6019866e60f5b753fce38de2ff40", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.199.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "20a9f6c80e3a0bee11d7e3e4159b4b6fd858a8bc7a479e1549f1e8ce15933403", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.199.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d6b24073c5fa11268d2a481e7e83ca0b9f9ba8066f3fd216d670f5df46a7346d", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.199.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac5a80ed403b08b7404e80dac7eda23e7ce5d5b0a1a2db36ac0b6bdc7f7225b8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.199.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3449349b395048f428dff7a08fb5b18e28c5f1a539ddaa3a1caf7404a88df260", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.212.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "30d9f63d7b08ba3046e43298dfc76bb222d5e6909457e8925f2866eb5faddf48", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.212.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d4af60d7a81f141b6710a95403a01f872ddf6420cee2b482aa7f36bf258d1b53", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.212.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e09b2a41a0d721e1121552044fef81f7e75fcedcc9c8461a973cafafe67216f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.212.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5836bdae40596dcee68f3eb409e8d0eb2e1bbc75fada210cfbe93472a5f1b515", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.212.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc083958e3112556bba973c1bc36cbd178adc8dfb697b231223b32c834948357", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.212.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "129611541a98f7c4cc7c4ebcd762343e1bb5c424c87c21bd5bd9d078401b3f5a", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.220.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c9cfc9ca8c1072f85ddbda81a88e602ef81afc0499514776f4e871b502370258", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.220.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8160a25772e0c6a0f7c1e6a4a4d5c7592d3a84aae7953c3a1cd9f353bfe4f557", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.220.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d07560f242bd5ff47b54d54204d73de2f95d0f2c0fe8db10e22032372faa341f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.220.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7919590736db7db3c9d319dae870e7512113ff071e0d7fcf09c4d4bc0004fb44", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.220.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f0262db24944b333677a8fed55b85a1d6015398ee636092aca89bd0e604f787", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.220.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d48623a5822a2edc8fa811d26e9f89b48ba59e41925e6e4e9167d0872e659d6a", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.221.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6d24a3f5fd0d573e089f2cb34d83bc2d74b8fe1de83d1b533c1f64dd2851c8d6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.221.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "095688c62aa9e17ae38b62a67e90ebed2af6d675a4c435ea03d3d084463b2e76", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.221.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7cd6fde0cde58264f8e6fea9cdaadf84496132ace2e33cd0dd67460bbeaa5115", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.221.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "614b95afcb3761b97e6d23e6570ad3a30ff86522f3a0520f24d6fdd4b89d48b9", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.221.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "69b3433323fd6607a345344be1fc092b3e6f903a85245b70b6fbd0b6ab07c82d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.221.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2e78f8b8e711c202d3f5eafc7a3bcffee3fed01c3265bc0ab9fb93033dde5273", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.234.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ea62d0adcb88e312e270197b959c17de1d3d58bc3d9aee96c395bc0c6caee625", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.234.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c2ec8d7a0acfd6e113d15465121ca51c79ec77d6168a37566b04d4845069549b", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.234.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9796e58d098213636eae26f411f9f6ae78a4673709742d46622839576417bba3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.234.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a938cbc964e6823416c759828954f4e62d6bf500f000a0c4c8bd16b02885467e", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.234.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9721f21af1b0935d31abe1892917739d4ecfe68abfc20db5f46d8ece43a00e3d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.234.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b91c95c8307dcb4d992800bbb483f4bfc3a45065ede882202135e487a82e9dd3", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.280.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c92618666a4c3f0642c9871518f19f9c60fa35f55166cdf5ff01bf3a9b32305e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.280.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "968cecf257f8bafb321df5f04bef2817a8522de155579a80584360fb5975ce5d", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.280.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1cf992dee989e65667120dc0acbcac5365808fd3bd0bf42f9bfde88be5e642ad", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.280.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "957777549994d27134f03bf1183841d4f2f672fab0bfb64aa34b79f90b751c50", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.280.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b10bbd0ccae2a0285acf792d04f8f097f88171161b7589b653f93f582a02ac5c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.280.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "628551922c07da260a269882aea7980838f087ed7853655bd431f450c06c55c5", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.31.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b0f8e9b14e532fb85792f19135a118185c400137b16a99e31a04585613e6e5ec", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.31.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "72b43250b32e78b887e186e10a8e5083b4fc429df9beea89c38b6a7e16bb1bb1", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.31.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e9c76d5aaac92d55d4fa30d3ddc984a995ef3bf171ed0e4f701d7c931277164", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.31.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cb53041dfd121a69cb41184feeb7bd2cb7e62fc77b5b3694bca1b7ac66f17a88", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.31.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "048d5413dfe40b4c7bdf1b5f22006797a1d24bea1e9c44d0127299af85fb1723", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.31.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8920f4b33299dc0b9b43a6a550ef4acb8952128780a4f29a6913b126886c65f2", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.36.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40552d108c74f1f87df6f5f61276ae1619d3ac45a4af271c026a30796d12b188", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.36.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dbe9448cd509ba9fb7eba5ae2d935c6073c5534b30af07a743ba8729039aeb24", + "shape": [ + 32, + 16 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.36.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9227073318e235aebfb70346742c5ddeb3610acfde5e66e1cc44041c3d15073a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.36.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "23e7db3e6f6d980d3a0526601e0883ff5a42071ee05de49bc1fbc5cd169d1b00", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.36.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "50c24eff255d1858fde8c3868a1557135ac54d0b3cc8383c4b4c3944eaa17677", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.36.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "284c39d5e02fcf77b355a56cb442be0368f7e24301deb86d57b69a51525c7f73", + "shape": [ + 16, + 32 + ], + "shard": "model-00041-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.47.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "47a529bcf0bffbcc41ae391f3aca5bd149d3d61d318e7eb80150055ce45e3652", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.47.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1db0d9d433ea3335e458fec577f41fa15f15ff3bd6bd38bd6908fa870c7e5784", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.47.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e359edbd81b82b976d1d52d6497c2fc5df23972e0c8c08865e7627be61814e49", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.47.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e60620885760cf807da57d26e49dd7886132b6620d19362f1bcd0643f6bf85b", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.47.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f4d7c4b0f5a8faff98c31f39945e598023c1737a16d6b919f8968d59a8fff829", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.47.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "930561e83d2a236255b18b54af22c0f198c37494d8aae5f1fa53c93474b02c84", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.7.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ee2ee4e94ea2af568aa75d11072f8a48fef460c67e1f2edc175fd0235681d30b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.7.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ebe967b218c71ffcae3b7604de59dc82750df36cd8159ebbade9815124b910e4", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.7.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc8f02a54b1fcc4cbe940de94bd396e69da1fcc0c12e4331028671af3c7788bb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.7.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "572c70fea8a6bb0218b452fb99c77dcf2ac79e3b717c5343700c95a2a49668b3", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.experts.7.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ecbdd1f3588ebcaddbc4fcd83c222271ef5c54b3446bfecdb0ada7f45263cf82", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.36.mlp.experts.7.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "042a79b0e12906b21bc53d22f22e9a0eff33f62f6e7417c548c0d3e3fbae84ca", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.36.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "888a96783ee7f389a3999f913ad6d74d4cb41ab7cb9c7f54d9e79dc672e5b572", + "shape": [ + 288 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.36.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "e6be06a64e7a9417b68c2ade6c22a22b434d244f1b6e46ea118ca6b2803e7660", + "shape": [ + 288, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.37.mlp.experts.134.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "80cb6b02cf3fe3b226f8269fed071f8868729f9c5ff830a84ff6fc2349ed1283", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.134.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b57acd656f349c7deebfe0ed0be6f0eb51ba97cb625d0872bfc916c987e7fa3", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.134.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "83b0b76dfe44c6e5e81aa4b21ad3a69b8827709c614190a580a896fe8bffd049", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.134.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ff2cb5f268995beba4db325c72c019bd447406b69f3ea0784b23831faea1f56", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.134.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5dba6509a1a57375c053456a51619a6173cde5dd5bf59dd83b88316ecbbb4f23", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.134.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "025988e694a5edfc006eb2c313ff52a93dfb81e13b90e0d60a4c3bdfa1c98d94", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.164.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6456a228d4affe72c1f60b0abc7db8c70997bb777f7e77f9a73be01c46245193", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.164.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae7e83f3582ab512df374d4d5377df18644edf1549fb0b1d417c692dc9e650d8", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.164.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ccdff9371215c2fab4eed0fa5171f964ec2403f8cbe55147e8845ab3b1320336", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.164.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "324af5891502c29c3f65b820c95244fa343bf2a18768fc30dedb40a793c5341e", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.164.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4193f53974368db5250d659ae7705592120cc1cfe76e7a89fa3306ad52f43345", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.164.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bcb634e4f3788b21ed821b9f4632506f5d81c5c57eec06ed58a8238a507410a7", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.181.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "38bb0c3eb5d2bfbe5006da034f26dc1fee1e20ea51095e3227ccc7ce471e22be", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.181.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8a2a8995b8353f70d5edb2d2af1c248fb64d93e75a9e290d6710532a12c353c5", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.181.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d97594cd3783e982a04145e244e36040854f614c4e095e40b1f27e7658be8acf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.181.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3616a85c42a6c3fb8f41f40cda6d6142b50f0452101ec47cb662692f97747995", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.181.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "168cd3569894795b208183af855122e328f9238d0f092c463e01ddb952a835cb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.181.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6a22ac0147c3aee14ca32e86220123788463a8b1c0bffe7339720478f5f4fc9", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.197.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cd72c50856e0efbea8c2da40f699eccf3d1971331f958f899c0bb98d58a0cb36", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.197.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "500318050367d690f9c1010cc9d6667aedbc25d61e22666c83d5fb3f6e2a1505", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.197.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1eaa73a60b55cbcf697d004b029f5599c49e3fc364f202707b486f54517500f9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.197.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "103c233060bc642087e5866529842bce211b776c22e92854155fe44fea000d23", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.197.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5919f7878a3764807e2ceae82dfc65719b8054d6715e09a742651982dc5f2627", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.197.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01523409c56fb0eaa498fed00f3a7f3f730a4921ee3f21b3d06b1d04bdf3bb50", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.202.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7a2b6a1e745736a5196f09784542433ae855240b3b833e9cd7374fbe2356cf28", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.202.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f380632760b506fdf8e198315f1dc35932c86d758fdf8bd3de40ada1c8ac4257", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.202.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "54d737c9b480f8de0af575176e029468d0d989a6ec6c7caa889e2acdfbda2e57", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.202.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b0533a41886a72f2ceb8fb0b3f5e2fcb66530f7ff0d112cb31a0d8fd15c3fdfa", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.202.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "70f84408ddae27aa367f6db126dff2194a7128142ff3b80ec774fb26f4b79a5c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.202.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "42117b1d5e2bdd16dceba51e4bfab725739ecfa97e8db7ef263dfbf6bbc12bf3", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.207.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8347afcbac1480aae534a59e777bc31f00c4f8b1d22eb0dd838461e53664367", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.207.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "11f522fd92f1a9eb9b5f1b500d5db1100469243b1c937879567a54fbf2278461", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.207.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "001457af630347cc542e428d549ba009e796d6aea91f868d94ed42c9a68aba91", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.207.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "02ebbd35a12d225369b6f3d3e248c6c55e769862ad2c07e46be8b4f588f633be", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.207.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f15fb89635bd23b00ff1b68188a67088f0157015e6dee209ba724c39216e4fce", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.207.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0e3ec18358dcdbf4a957d7fb2808215797fc952bc53aab546490fe7915283310", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.214.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "422c54991ce317d710abf3986c22113fc344539a93a4994bd01de372f65e0585", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.214.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f369fb98160bfb74aa27176015462cc0cf848dc5dd5e40cc8a27b7e3badfe2f5", + "shape": [ + 32, + 16 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.214.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "51c18efd14cab7ece93e4d076a214a03e10e2270fd56ee1ac523dc5c35465229", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.214.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6e460670189bdd3aa158819372589d0739f87f90cfb0d1350914b93e2325103", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.214.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2412ce83feff7171bfab96c190050b00878b123dd14866d96d814aa53dc32f47", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.214.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9119d1c1ec177e81aebe9d2da802bdb4133b73571399fed6a528403aff3bb6be", + "shape": [ + 16, + 32 + ], + "shard": "model-00042-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.244.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b4b7a5642b4853d11af27d590a7879cd6e94a69af52ec38d19f72b7475cfdf10", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.244.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df4f1cdd717411d92a29f4bcc890731235a4cd657738e57e7519ff277ca1db75", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.244.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6821979b108f00933a071404eb94a37756f9caf67c17728049ada9f17ff527f6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.244.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "60fac3c455e76efe4973ce2be79fa3d142f84b8122dfdcbb996110bd47a814cc", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.244.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a4a00a15bd5782bb3961cd134cecf8d6fe33bc6515e9638dea6cb8d3231db45a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.244.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d1bcb1bcf5ac12b068095f795554a603f7adafc7bd1598c3c2642249d45adb5", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.264.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "13900aa0e0c0b889b13bfd8014bc4e0cad9deab17c0efaba3c67c6cd744cea8e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.264.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ac62198093695d58f80f619d0421674c11bd7ba6906684176f0afbb332c2b54", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.264.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2b354f2d28af4110700a636e6e6622f64f7af0395ef8c7e463e7b6c807e4e1f4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.264.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dc7fdf077efd5f4e0f4dda49e381c5673c1101122404034ed6a30d17d8c1caa6", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.264.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "db6c241740a8f9cab245f2ed0c2c09f9561cce7f2ae67ffda1ce8fbaea77e728", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.264.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d4e483cfa4f4e62f1e68229b9cdcf3454e6aa80a0c0c0311ccba22da4f123fa", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.284.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f1078fb446f04d0b768d79532beb9e97bbf3119e4fa6c25160f6753f59a78436", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.284.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a32d52da6d43a82c73e58c0cb3dbf5e54642fb53ae1275d6336506d21a00e086", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.284.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bb301d0b4f9abaa332901f5b634a627c2d911d4a29ec9d858f1cf622f381e63d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.284.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f00e0b7ecc9d1fe324503a83ee1db6b354e7236aa44f8c02bb46a9b5b0d78fe0", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.284.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "de31fa6abf1b3d06a364c0a32b1573026a7c18c4efcd2de928b7fea60fb0af96", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.284.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7598243b882fe2fbebad823c57240b08aefca05235a4179103ff79d544291025", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.33.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "69ed350f37cafb6af626d72f29ca2d58d43c17ae0efd92aae0119c5c7776cb0d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.33.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "30f42a4750c3c2243492a3e2559664d56c18e3fd8e66af5970736aebe847d0ed", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.33.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a3bd4bd347d8990ccfe60b9b98b58ee1dc698ba07b9e89567b60c40375ffad2a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.33.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d61626a80e472ab19226ff9a56a057bccaa22639d24416968dd1e8ba703183ad", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.33.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9865cdd60af7e31e034320039755d0c618de560149f50eae674c3b9e16a1361e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.33.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f6e94e6221b1640035ab364d04fbfa048dacb78d3eac456957fe71e910023bc6", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.36.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e584acc63f186a7f037eab5da7dfaee58996abe6556a2cd78ed46657eb349f9c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.36.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f143a9b138de3d1e55b52cdeda56ff4306201f90251ba30dd6445c1937da3a14", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.36.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "66652fadd0decda95708a78da84b1e01e5c9427549a2be99ed203732426e472f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.36.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4858663996707aab5489872b866d45c93c3041e25ed08688e3f1edbf8688f383", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.36.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7c1133fdf7acbfdc7e74d35e2525785e7e52eb8e4627d09ab92ac62f37dbf828", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.36.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "418c55b21e7ee0fc81c90c66724883aaee3d3fe3d3400aaac05fd4db274b6273", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.59.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a80575120fe476959c57291caff644f02a0ba0490bbdd6f3f424052ef45eaf72", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.59.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0c6b2f6cd749ca12bdfc7b75f5719bcaf63628d600fe35ae27978758e550c368", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.59.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ed82cbbd9d29c5ab9a2b0b68fcd6576aed4204638326a81ddea63b951e56beb5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.59.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5a78bfa296a11f2ed541bb7802385c593d5fc7d4efa6161980bff62f0f75b992", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.59.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8dce3f92f59ee11ff7a220a87ee808a7fdc1470cdd40d76fe48ca8f6054b1bd5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.59.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "992bef7ec05c271ed25ae24d0c52d45227396cf86eca17023ccae3ae04b2b98d", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.71.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bf58d711ff5a286f9f5cfa4906a399a5ea918cfe2cd0c5a1c265e070bf4f4192", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.71.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0d423d90d28bca32d58e996713f2d78ee8681daf32a1091c6f4d9cf1eedfe05e", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.71.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "352ba42928a179fdd11f62757da4382c4515b49541ead682d25b297af7db076d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.71.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5aecb13ec709f8c791ae28edd59ddc350cf58284f22e43872e8cd55552258f8d", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.71.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2ebcca552cf1051bc31d32599e78a738b0f3393ceaca962bd8860b60e580c4c3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.71.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3c660ad70fcbd3e91b209d660d25a9ddcb5076bb02218ed2d674744aff56f578", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.84.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d51f44c3e60cc0aff96c61825e1760693582f0115eba30156124cee99a748cd7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.84.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17b2fe68affda9ade563d7ad079e935991ac524b0ea1e74c8cf07e3cef6d89c5", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.84.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0bf281aa460b063adff9e97ecdcb843168a4472368aca10ee649f7d4801ff16e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.84.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ecf69b4ca8099380b147a3df0972d0a6d1727983b6b853a99906edd113b5c281", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.84.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3336b44205fa414d61606e25d9d12e4a83a6f96dd01786b56d85ce0fc04a81d3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.84.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7f11be553b230f6d798b66c2e6b26436b99c4f38c381ce22011fadc106482b96", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.93.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "39189ac7d400991faecddc063d83970627214059e64e1e7b3c8f10a779a12bae", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.93.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "610b193e82a70547ea65e39529ca9a11df673dd4697eb89188e93ec294e8c2f9", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.93.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e053db918743146bf01edaf96f39db0670423d2fc3e8d4217785ae36008ddf62", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.93.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d48d923061101f5250f015dfb852a9b1e49cb8661f997a3443ef8471409fe031", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.experts.93.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cf5710015e22ff69204f018c3c63dd1d0242dded0043f7ec174acadf2feab8a4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.37.mlp.experts.93.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "24ab900981330322cb8746739d1f2b650fd6de60e9d692dacb283557c5cc7d85", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.37.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "4ffdb656b0875830d91d4ee90e1606bdbab6a1d6c95911ff72a7665170813e7d", + "shape": [ + 288 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.37.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "04137cc6d0d4e98e1ba33af1c0259c3033ca9a1edc15cd68a2c49306978a8c48", + "shape": [ + 288, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.38.mlp.experts.107.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "897b88f872b88ef2fe8268f5eb58b9a7caa2b8da580d606bce72837f6413896a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.107.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4316ff79dac2794eaad1c4c042db3df2d430d71ac30faccdc1355dca60b7834f", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.107.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "17b752d02602b5b31ade6edd1ca51ea377c158e010e118b0185b7ad1ab0e0696", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.107.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a30496c4ce591c2b1db98481b49cd8a04b3c1f8505c13bd13a87cdaa1af5ccdc", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.107.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc16c695013cff9da70808fd49c26176f1f1263d171caea12a810392323d42e3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.107.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f156edeb180c60c96122afb96ff52f7a736e6e6ed2abf401e830b6188dcdc800", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.130.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5f65efe2a3694ba17a85a55d57a3797558e6a576b83031c53b661d46aef45dbf", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.130.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "509e4fafa706a24aa59c43c141fd29af9b2f1d317ee52ef7ca8006e037a200e5", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.130.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cca662bef1acb9aaf8664fcd0febb89a11d470031b7c247d941ba85631ed3777", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.130.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d178c8e9cd5727842965114974e9b59b69d2e85f1c64d7da38dd75a7cee33cd", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.130.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a39ec3923270f99d47d17cb1b0a36e6eef7933a263f8ebfe43ce2055ffa665f0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.130.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "541687dbd422a0e047a527dc641c34660f20c568ae5e1b6c2a1c2e555da4788f", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.14.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1345848a090362f2536dd06b0543aeb32fde82d5d43990c441734c533b4ad23b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.14.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "605fd6e16211b1627a0ed71199e8503d1167b2744fdd1b119fb5ed71a0e07066", + "shape": [ + 32, + 16 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.14.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5310584a3cf5a05471038b7a4883403350e85020868440c6d06c187a010860c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.14.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a794ef6262c2ea256d91b9b2ba81b1855a6598337995947ab5abdafd999088e0", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.14.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e740860842a9d0e8703889b94923ba47d672bdf67395719e0c6fe80724e7f2a9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.14.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "61376370722f677224c6535797404aebd8215448ac1ac5f2435c40a2b18cce7a", + "shape": [ + 16, + 32 + ], + "shard": "model-00043-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.143.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f3e4c3e6c667b450550fd63759192fcf783cd18f94965fced8105dc418e9a9fe", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.143.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c95bb1a8b7ce31956e0a99ead8e433368697eccc75eb0b97fffbbf1f1870a9a0", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.143.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9f1e3f7beaf93a11d7c18a5991d5993c81f0471ab6ae3ff6eb780ec7dd10cf9d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.143.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d2b6c8602c9d4566d910b6b3fc7897b878af1ec8c6964796afb3838f7a4c05b8", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.143.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8ad93546bf71ab5f1b95e2534e46504fbc54eccd19e23b710c7e6c0395d685e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.143.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ac8941cacea9134d87d91ab5e40fd3f47e0984ceb522080defa83200461e3a44", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.15.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e8824ed0092f5a67feac608566448988c89b79a9b067733f101fcabeee03dbdb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.15.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "90150bd4ee39b4a414a464d419a7a4cbe6bd2aff1dc2bc595d8a9fa3b5957cd4", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.15.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d30143828ca86ef39164dc01a1c39a22eb56fc96a0e8d7940c36fa8842b24bef", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.15.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f8680acb3623618d6405a097111be223966aeff66df88143b8581922b94ec9cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.15.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "be3195f4c051dac882b768b818149a9586b6c9e1fab9c1f3ba51302c9c20c2be", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.15.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cc971bfc6eefba5b6ee8b018fa4bc5284e3fa1768251df438e7f3819ebe96e0b", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.161.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6e8e0bf15724ee92701a3556ea367224421e1507ca3ece99b61c377eb287dbb0", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.161.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17aaec523f71bb2923aecbc36aae8146231395c1fa2c31ae6a74de2d050d7ed2", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.161.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aab4c82a5722f34a827c5a737866551381411724157fbf682f48d7dcf74a05a3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.161.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3de17510ce702372682fb7e02132abfda0c8b8fa73bb52502568317690fc1034", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.161.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "438555496ec607c97078cb155b3e2ba45a76792cc0c8eeba4521e9410a11677c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.161.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "430148d509adc9ff77f881aed9b4fe51aa4ec370597c0981c2fa26485482dfb4", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3e35592f6e1fe01683731da38adda8e883766085e87e2957d96a58d2365db8e7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "77f54594c2d4c2934900d6eb87bc707022e9847cb973e2325e72da3993606d24", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "45f52d01267e6abc6e2cc7e1427b838ce6a8c6ad86ed67e3bdfa8d5fa6749279", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "66e1ac71f206a6d6b8cb6912cfb998c0ccec3da621022a8e1fe94816f96ad63a", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9a33e8d83ba5220bef86e8db728bc520010eb01f701a1368c7db566ddb74a20e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c1c3674972ebe9d3a98417b3f4088109139db712c290e2c8180775ccf637aad6", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.195.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "90bd0cb58759e835a6dfd0b1b5676f8fd28e31601f0119d4ec74d3b942b04bc1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.195.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "94b6c09ce7621f6f0d8b201f01b1a91c445447350f8a099ba5e20646a7586f49", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.195.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eec92aaa53b9a2fa7c6d5ec546ee0162eb611836adaac53257d4da480b562d38", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.195.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a5ca4f5220428c0648f99a72fe94072092c8cb3f0e2c7a0e9c65445d3af650f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.195.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "19b69dfdf723cdf19f5e87a50dddeeba00cbfa4a7cd609a14a3b27d312de42e3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.195.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d2582bde856137d4782d671e932e5ce9f14d6ad213c4fcb629a4f9721bddd653", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.212.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "436f086fe9119662df1e23b3be79f390095a677e154bc70bf485e720d39c22dd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.212.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "225e8a64ad7a364bc970e0fef2594f69e688ef194bc2453078624defe580b39b", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.212.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e772186e6f4aae7ad73b0efcaf2116a17381c3b3897e9f2b2cb84bc56e7a910b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.212.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a171636c278c3850f0fca01feec1cde6cb0203a5a139d48751a63e7ddb5d599a", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.212.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dbc7f59210198d9d1d84ecacb67c987f90120c8b6e2960a67ebd89382be9e123", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.212.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "be8f8e43c1f12c63041cbc697f53cc5a761b0945c8eb26399916f56ce6317aea", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.225.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8948c5cec40858d834eecd7e643cfa1b8e9dccb39f0b2f703e771fce2d4df5a3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.225.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "df02eb4c646542c8302f2853b0fe3dacc172236631b6742387e43bd7a0f205a0", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.225.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "71d9bba06dff261ae201adce772519c3156cb4ee127a65294086af2db8de3bd2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.225.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb022bd5e94363280aff94b41b9b4686b45e4593d53059202cd41e867eb7bc22", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.225.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "da6c25746223c9b2050d4796b2798e15aaef6715c0aac37d2ae83e3de2e5119f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.225.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "212e4c21183a08ef20afb44ebc54f84ba2da9a294eb8174aeb0084eb00f08e7a", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.258.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "828e0128954273ad0191bbb9a2edae95ba5e1f5d4c7c6d7b5c19273f66099711", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.258.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0069bacd1ca295ce97b302b3b3e0bca2c66542ace5e2cf6ba71faa7c73fc6020", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.258.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3f83ac858b869b168017dd1918cf888da79faa8c64d2cd7af4c1e3f8b840146a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.258.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a877458cf9ffe48ef1cf70bc3b2c08c9aa2a1481cd7af4c4f314f2f1ed543e85", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.258.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "173b0b056a49cc19681b50e0174207402940c9402822ce86016175f51b00b893", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.258.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "352581aa9001bf6365f244b6277410c4abb06a5d2c16637e4ea4e01d71446b0b", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.259.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cb24186c4eab712754505e5ff9d2518a116f6dc098ff4f369873ce918e7c07c3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.259.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "312177940aa7bd2dad5c127e0e7321f09f228212535510888caa58464474addd", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.259.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b6480d2d6d346fa8b875fa4c87cf905d10eb053f6c41245c4419690fdcd8f421", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.259.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ec63b12f0323049c0a8769667f7eef308136b312449488c1d45921bf70df646", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.259.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f13459eaa683fd929c7d4485d29ec89617d42c0296d3634bec455111db8cb859", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.259.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0138b03f269ffb6c162d392841c9e5908024399976804004dadb0aa934810142", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.281.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d21ed497694b90950d47b3cd60270242a86451e1324ec3886196d0259f59a65d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.281.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c7efa361c550fcec1053742d2bca66da7a5ce714bbc98851f9468471fd9dfd19", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.281.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2c1184ab8c25eb127062d164d37012e32fa5dd30f4561dabb40fb2d5dd919e0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.281.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5878167b7013c939e00770b6209dd547709c6ac9167eb4e4b64fa41b3b27d75c", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.281.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "78d39c636b88de882a0802182e13c225c54632fc8fd85022249625a24df735e1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.281.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "35d8f0ec4ce35cfbdd37bf961ef83d6fc7020b31798e5d8e85269f666678e785", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.54.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4f31a500b7d3e2dbb9d652dc7ad0952771f633fbc0396f0a3db11e8294bdaa9c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.54.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec30fbf4d5b5ba76038d761bbea0a1b8a674a02aa0b5f605dd3d2e8ca0bd0a55", + "shape": [ + 32, + 16 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.54.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "da73ba0e8209a423cf2d7ad4c01f19cb78ca55c2ef28c3396fe05f9591f5d490", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.54.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8595e0db88bb77c795242b2b362fe50aee4f2963cf8803053265e2587a4a4762", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.54.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3c8078fc168db86bd88f681f28b9c8e6a81aaa3944b698a1ef0225ab7b7057a2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.54.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9dcd051724843e90c1e6452c817df5fa0835e321fb591ec795ccbf5101d6cd49", + "shape": [ + 16, + 32 + ], + "shard": "model-00044-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.76.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a3798c64236e9ecd52a6f588a37d327dea0cdbcb06b033a56150c0b012447336", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.76.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7facc85a1d79045aef4c3ca066fc194efeeab3c55ce436fd05b2103c2a865a96", + "shape": [ + 32, + 16 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.76.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1822459a583bca29fe5e679dc80a118988f695ddc693adb41c27374f8fe08c43", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.76.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "55c47f3a165c8870521ca1f6126fc6c09b6390256eefc6c6a500bd766fba1b92", + "shape": [ + 16, + 32 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.76.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "69ad752f55f47331ae09fd393aac3c7af637d0569ba0fe44f3d89d15828e34af", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.76.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8bc604718cf9a7f5be7e3978a814bbfe6a4ae0755f210e820cb919e1c1ee9b7e", + "shape": [ + 16, + 32 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.8.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "452550539111e90933c90774a4cee07864f06e9dde5a4590174ae7aa8fe1e5b1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.8.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7b69ca8b015f396ad90e4658f33f759f7689199e48b2b8047ed712eb33e8221b", + "shape": [ + 32, + 16 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.8.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fc91a29997b5488d8aeed1f18472d5103e6a4cf799c3b0d081b63cb718d69102", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.8.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "529f53fc7fd4c8786251c8407c5f4b090e01a37ac2102f4d507a5d18bc728d77", + "shape": [ + 16, + 32 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.experts.8.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "33022a0451445c954e8d14a6602a811415dd2c020c9b29ccf982a436377568b7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.38.mlp.experts.8.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "894223107c33552be925de307888b51499369a99346108d70a61bc7cc2c3e408", + "shape": [ + 16, + 32 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.38.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "3788864235f8e7632cc62eda64c66d23b5020c3d5afb29f3588d459cce177035", + "shape": [ + 288 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.38.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "7582d586208ee4db7b2af1ef7dd86e9d6a4883daa1f952e7e7ac510010393866", + "shape": [ + 288, + 4096 + ], + "shard": "model-00045-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.4.mlp.experts.142.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a5ec054f68689c7fa809cacdbcbe1b6ccfcda553a21df4c2049954e46a2d2ab4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.142.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d01b08887c08ed972cd803ab58e32f641122a9754a3f739d8f51f901039beb51", + "shape": [ + 32, + 16 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.142.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce2c8a70343ca66d524e2f52494d6acff7b6f1d9d2ecd9752c1668c5bf066be8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.142.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aca04cef8fc7e0f0078299c74c4b17dfff2c65ed22633453a1351c760e64368b", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.142.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "86677ee349508a24e921c78058ee9e17e2509b9852a1623350aad17cb106d217", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.142.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a4fd84c89d05bfe27e2d3606f0afb5b6cc924b88a929cd18402fb023db6c5cdc", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.173.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "da760a193a449b6122524d0c77a77c06e383520a3f163a2684f58a9e928f8c20", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.173.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6e24e3842fb56adaa320422fc7748f009e051761520151955cca141e1ee46019", + "shape": [ + 32, + 16 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.173.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "37a9800ea61d501a47861b25462dad2ea6e7ef9b697714b71db6ddc07edc2805", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.173.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4c47ce3fa354a251db0c3ebb6d33ff5c3a5c1792c53c2a29942fe39993ce8c87", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.173.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "689755e1fad7b3485e63e97c379308d8fe55cad8f2dd1e1b1ef03b987d2512a8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.173.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fc66cf45b5b518465e7781db7de56e07d626161a97519ef9b5651bfd98667bc7", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.178.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "df38105cf7457724503cd512b8bd2a02c04b321dd1f6b28aec5149ca02ca86d5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.178.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1efed58cfc2d290b97ddaa1f66d089c3ffef4ba9f4e75a49f94511d2d591cf89", + "shape": [ + 32, + 16 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.178.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5585c014edc0b345049b50e5e9ad1c89c1655d2ff602e94d17d46682a43c970d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.178.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "317aa2573d7cd3681432e44726403fe1deaa121819589cdcfe01a5e445c25474", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.178.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5718f27abd9af8e720bee08dd06fc236b4667fd9e43393abeebf2e9ac6b8cac", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.178.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9559dc6b1c4ea9c966d6594b4146de1d6ffe6c9ee2a237e7b5bc162901b1b283", + "shape": [ + 16, + 32 + ], + "shard": "model-00046-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.197.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ad617298007431f3ad27770eb134385c11a399df21b660587559a29b29c623a1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.197.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a555b2f31d7f960ec7513c4e2a657c6b0e1aa2d48f42fb57fb37ca8b215c7f8f", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.197.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7368aaf0eaea2edc4b33391d282a7fdf8e61c590efd0eb30021f3e431778d5f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.197.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6e69b42c4f1a4a029959a4efbc4e7e62ced4c57887229017ba1c14365ddbff3", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.197.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "63345c257236c6e873ec2fbc7d47ddc83c972ad3e81b64446a40ab39aee3ec90", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.197.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "baed562f2f6730fef2609e530d640bd893ad0a5d240b9807c73d893ce50b7a48", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.205.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6fde8d8f174698599629f92923e7ebcd5fe9bc86c22d7b1192e677672abd2da1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.205.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f4765beed3b6862c3f83a8cfc446b2b063e4abca2ffb71048c8a69786459f255", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.205.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "537dfc1a63e884cd46c4164bc540b7c0e2183b1f89a9e750cf5415baf72b7cca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.205.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc57a6a553f5e28f88446e5eddc40f5423df135cb78fc23fb010a4d7affa68bc", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.205.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b6a3500125c6fc13a45ceb8cf465b997d2d8cf40460329f73b4629b6f8e5bcfc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.205.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cc6ec13b31d937f2e1af2e05a478149daf7c8a84995d3231e1a9ba151e2fa25f", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.216.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b765ecf5627ba46e56f8d0f7bcd7d7f92d74911dad82d423657fb14f1ca7be7e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.216.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3adf07ecb19263ab334886ac54c76db58c5f6a9e8aebc348370d371bc53fa1f3", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.216.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cb799af9335573a383d08b31b800a5636a170dbfb6123b1f35751650fc0e668b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.216.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "08c1b9c9f628c88152d753d2e4caa272538632beebfd97554e5a3ed22a213ac3", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.216.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "316d534a768f9bf73355b17bd93cfaf2b55a849e019497fa6491eefab78f5f94", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.216.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0645f26861de3d79e051b0e722efb4ff4b8f17c67e1ef3e62af7ba4a9371a87a", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.223.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b76dd15e2bcde0ade9b9eaf46376f01b05efdab996f118f08d98a0011a3bdced", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.223.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c73e6d8c1baf9ea3fa6077b725e20d0ed9b55b2e39f64872b1200c81899bccb5", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.223.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac214014362a59297f766cc2e1a07ac3417d09c1deec271b9d564450728cf1ea", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.223.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b276dfe67c5404d294cd8e6f9a2958633ef27ff10b558d566feace317cbf63c5", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.223.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "de847b5584dd4d220ee720d6187dbeb8ded7758c879d9654c0abc98283c8ac05", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.223.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "624f44a1ce9450af94ad591f41ce6ca2d7b9918a5d2f7717b06b9b8afe66cfed", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.241.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f3d27704382eafdebe672766a7963aee8c475fba18edeb23e3383bb8453c2b98", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.241.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b17bb78af453f7543bda171928d0e4b316cd3b7596736ac0708315471971c88c", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.241.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "848ffe42fef9c41dab0f160e20e3ae4e3f3d6bd878b53475e3beadceb6e190ab", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.241.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2a925e849b72dd3a9087648bfa811dd11a90035f0d4a95395c5fbe7c12db54ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.241.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "933825e3f455abe819aa9014e7093bd25abd1a434cc3ae181e1ecc99503452e1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.241.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7f8aee640c79c050665069a74310c6be5a353b33362ba9fe9f0375e5e9a88658", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.243.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c1ccd32da6032e21d51bff88c8f815fa8107058e77014547da1452e5ded361fa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.243.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de344a206a1ab9a1443ec31ae9158ef67ca90e7b79440727666a5b9fcff92fbc", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.243.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b5cb791930211be399eaaceb44befef24d47445ca7afd1bb1985d7e2f00e48d6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.243.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4c05990eb4fd39346ce4bab1e2f46313a08a8102d54711ecbc050c5672cf35e2", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.243.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0e3ba0c9bde0e6eb0eaa2d42953c88cea573d2756bd71d076d5c76d60902651b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.243.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "25cede2917a3fe6ad8292b4d8ce79e0340ee076cd656e372a13ad67c89ac18d5", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.248.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a0429f5b0c04e4d0fa5d7fd7e1dafbe82b29c720090cd3841313bee1994298b8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.248.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "20cfa23de0bfcca92baea1e26565a89eb1c3896dd24cc725d100dae0a221bbaf", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.248.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5905cf3bd6d5762aae65843932c0b9c6ffc63a8090fa2c8b476d4afa3c22bdb4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.248.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "345c9f3f832d5eee5d0b1fc54ff828832ea09c2b8997b1bb8fbb3cfdb81994c3", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.248.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f7c4f1587dfaff5edb2a4e9abd7400ea1db279b59e07d29bde2d0fc35ac92671", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.248.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4f0741bd9cc53c5203e5088d4aa3d3f1f1d11884e36ede9afd84392ebfebcd3e", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.268.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2253365952c966c247c468ca1bb291380c6be8bfaf921eb6fe20d6da4131b14b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.268.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2725d39fcc76ededc600efafc6d62b5e670f437ec8a25980d9a694b05aa8d72d", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.268.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4c2523277ec41ff9d76587ee940a16f038208cfba1b24448d771264ba0b0f9ff", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.268.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6edb67a5bd99e647892d154ed2048d052f80cd32d3de5eb0af5e0a5f653e2d81", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.268.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c27feb6a5f588d6e51027278b135877bf707cb58bc273c0e7a411695c297f965", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.268.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "453085528f0f473aebb12d44b395ee4f56f5b62756af5150bb4af72475a7e98b", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.39.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ef4eaab0e997324c228d92ab83ed25e170d3a272548ecd8917037197a19977a8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.39.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "136bfca61bc2f72a71f93777e3e87436f31f19e5ce2f858575914edcd970b80a", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.39.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5ac9c6770bebbb6e73c5279e82cc02d530b7f12eccd5f81ce5b8a144d07dc75a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.39.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e14fd1277a0debd2b52cfb7d793b0c38e056834ac6d9b4cc75044b16e290338e", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.39.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4e9aa4bc8dde4dcd0f4e0e5815cb6ce5aeaf31f7997efef17645d19b9ad75d64", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.39.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b25e02f8d0db1fdf4862e52263e4741cf5001af44a5206b5f2cb4e9485ca9e50", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.45.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c1185b273b1799ce5f7a8d49d6c414c019ebe5856a03a5979e5327a300a580dc", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.45.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4d09fb35510847b55af8ccec47b6ff13cc4d7676449b80c4b6b5f1da1e78fec7", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.45.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f2be7021e35dec4c4a534414482e35d00f741f6f5d43e4f13c8b07eb4fe1b375", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.45.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0285f8957875ed3d52e107e65e82fb547c358747e99ef9e265b61809522b9fb9", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.45.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "12563b8c1d9b72b8662274359e0c83e853fef817187b363e6632ae04b73300e7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.45.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f3f917b2b752a665848f01778ed2eb2d560efd51836c69d3ef3de2b56bc5a02c", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.74.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d1300ad96ad2ac80ca93856947843651fff1466fcfe05726de6c346ba52ce06f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.74.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ad6ce6c04e038a0c740cbedffa74050afdc000c3e55568f33dddad498e809e8", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.74.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4952eca63e07d73024d6d02d484cc7d1f40351d2ac152de1419e23df867cbbd3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.74.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7211264337d21929730b5bfe083c2813329c8a8d320adfec26b36b3132b3e57b", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.74.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b5183cbe8a2ed963a7d763f458644591bbbb194854834a1513b4637e0ef950a4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.74.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "96be528f18707cb78d76946d78da40619c5b52321a04106c8bb74ecb2dfe676f", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.89.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8b7a2578dab39cdfa0c0f1c1cefa9043becea3697f11b9fc1d7d219a4649d8b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.89.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d64ea2a4a40be888f25d659253eda9af957e6ded9220e7b4e361c8fa20713f90", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.89.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8d7f5ab341362be375f4cccbca2ab342c7b312c609b3f9048f94aa15c9bdf3d5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.89.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c7f15fdfaeee3a61c82e67e44b175373eaa50f7b4a0e3a5ce86ecc7b9cd5ba9e", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.89.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b6c5aa6ee60a570a62a80649c0845a48c3500c781eb1b9af7e5981faade9abd8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.89.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "67deb3322763b210ac7b8a5623c6cdfb679b76254c9f589b9329e7050415effb", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.93.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84cd6c5ab27274eab9370e7049dc690d0ace18e4c6f22e1413ac0bb9065be28e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.93.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "55c48e2c538da0d2cb0235ce6e4b8ba29f145b8f67a1bc4514394076a169de46", + "shape": [ + 32, + 16 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.93.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0ba4e66e4e6b74ab6510e988dab0bff8762dba39ce1992f7c0736f0d4962d04d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.93.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e48022f2a5b0abab24c97166d2f8e82a2e98455281f4cac2e0577133054fa1f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.experts.93.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d8a16fac9739dab31141251d7f31b52d09846a12f819321483246b389c7aadf1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.4.mlp.experts.93.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "623f51ecc3cb6c1b080bff66f85063e53edbe08002a22b99adb590d6adc4264e", + "shape": [ + 16, + 32 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.4.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "222061fd96811993d3826f7ad4a626999da74768d618ffadce0f7db6d0a79b34", + "shape": [ + 288 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.4.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "fb68076977cd49cc6c44bebd2809292576d0edb05efb8bcecb670548f027f208", + "shape": [ + 288, + 4096 + ], + "shard": "model-00047-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.40.mlp.experts.105.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1579b35a789daced1dee773a90c04faab722c8f005fbafe1bfda8becaf9046fb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.105.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3c3aced5c31761af8de892444b493263c4d5f7d766b37272400074b3090400ce", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.105.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7bddf03c04e9dee4270fed2558978fa954e60a874d0b027deb946c753ffb1d3f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.105.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cdfcac18078ac408b7c346b75bffa21357cf68e11bd811d2347f7a63dc4403f3", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.105.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8399e2bbf094ebd2c11a26a90be9a8589faadcb064f9eda799a6ed88bbc5ecd1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.105.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "767a70331f781a1893dd592e3862dd090f2288172f5d9b495b560498fd3000de", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.121.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5e80fcd3d811f9acb476dd45993bb0acfb4510bdf934c963f3ec7a48c57eabec", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.121.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ae82aef211709300248381f9fd9d9f96cfb0accef287410ebd1df0269b35495", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.121.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "292859b09f2ee0ac8fe37ec2bebeb5f39abaea8695b18081b19d2bb65dc5be14", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.121.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b6fd79f073df24e1d24cc7f3c484c03dedcb0492c59d28ffd3ff27db62835c7e", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.121.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d0c0ff3937fdfccb96b3d14e94e2271f2cda2fa333374f88a364af581f87fc06", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.121.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7c9698549c598e4cb12116c36034e7125b986bcab6d71c9163ee51f006a07144", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.139.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "59295b5bdf96944eba2642ccc7dd4ffc430f1737312b89493adfa71b1398f3e5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.139.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7f738811213a9fa7c599f55f0274f2312c7318560d44e817875b4ec82a690b48", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.139.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eb1a01067bfe3cdb234400f6031cdbeac0f7ffdf35d16548ca0739ebd036f4fa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.139.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e729c0732cec236f52a8d208884872f513e0a89fe44ef33c4d601ac88c82decc", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.139.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2dfc830051f09abb8efed1eef803b09a8766680d2fcbeec872bd1579ced8f3f4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.139.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7170059d8063645e2443e23ffb74116028cf4e2468ff44888d377afa59f50069", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.14.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5444e25e47eb114999b2c61e3ec83f06678f0911b77590bfb37a9195752ed3b5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.14.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7a833bb4a10d31081d7dff30faeaf4a7e451a5dd02735e6f503d95b8115124a6", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.14.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fcb171cc91442f4877902b71c07530f12d9a758aa4e63e6c3471e3c62e3a581a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.14.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "40475d11ae4bab986b216d1b87872cb7736fd2ff758438e3df5ee3763d18b278", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.14.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ed74f8f1ff8783da0a30e014e3bbadc8baccaf79e79d6391b567196079ea368", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.14.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d1fae8159a22e158c63b39843ef1c14b6d3e752d0f93c1c85411dce095216487", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.156.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "604274f4aaa9b070e4bcb15c31b8a5a42f7b16293a78775ca6dc913e99db70ff", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.156.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb349b2d216ff4b0e0402623dd16ebba3e9a9f8a521b2eccfca77710c0363bcf", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.156.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "18264b413bd90358828c4c0c9ecb396e22012a8c3b8caed658e159322e3d10e7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.156.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "21c4d061e507e5ab41cafb26fd115bc43996f59691f19c89305c68090a4aad60", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.156.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac6a9f53a1b7c1d4271cfae03fa130c3e99e0b1ec4ecaba6b58c88c8b41e9acd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.156.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dac12099f3e6404eed79396efc0602fd7b1546fd8c887f4f37b42a42ee5eed79", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.165.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "93656bd7f9f2fe1f1e42a864bdbb29511195612d1370a40b6bd78b507d942036", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.165.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c2fcd3d813b4e3d34042977d64c86d4840841fa8aefd717e8313e5334522901", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.165.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3a983764b27ad971659a065d11683f02eaf12af1b6a89c536d2e1c71a857a3b6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.165.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e7fbc25cfe26f915cc6b22723f3bb1c9ecb658d8c2dc3cea8ba6753a5dec6d08", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.165.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6f464301cdd32aac6ea1317f0eff6b1c9e43ffdfc44d2a69368c45f4795e900a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.165.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "13d1a415dd0636d1e1372f0669c1c5969acc4de2449905224514c183ef77d996", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.180.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "536338e83098b55ae268c4eff4d1a43f27a03cae0f6c3178ff6d0abe92158f19", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.180.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5ee388141d6efc583280dfd04ff62d079136225489ccfd9a2f2b2a27321e1f38", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.180.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d5b14e932e8d9515e446637842f8918865031403ee4b4aff3b1238404b168952", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.180.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "10bf5f062cfb1c1a5e0e15fb3b597114a9759a36c9cdcd42ce93be7a1dcbf399", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.180.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "47ea88cb6ce7e2a8d7d5e2c5bbf8f30bead8ae4284473340ab0cfd94fd48f7b2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.180.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "120fadfa5c52201bf0822cee673251d19aeac6c6b313a121f035cd384c809944", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.187.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eab103aacc19375991d12384ef3e1366dc75ee6d21cb3ac419ce33c8a333140e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.187.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "90bb4ded7e75602bfb4fbf7b94f47430bfb9c875b3e107a6c80735c6afa6d641", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.187.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f988f749f5aada3a43478dd9fbf7234e7af85f82a8cb6803b276f52115ab392c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.187.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b5d234d5c3a69e231a7fea7c1f2d2e89e897694330242e4ad6fafefcb98ae9f7", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.187.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "83b96346e0e32c3a2d993a51b3f3a7366686bdb78aecd708d85407ebaf064337", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.187.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "22055be6c4235c7f8bcddd77fa67dde140f619c94227587d758f4c9d24922856", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.201.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "be6be983deb7b1160b3fce25bf1750182aece282c1c0933b2d3b48ac53f1ec2c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.201.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "315577c6684cf9fc3f4b07374e5a53e1403b3bba37c2f73aad8ff76b60bff330", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.201.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3541dc890415eeeb73e235beee8b36629d2c6eb02afc9f9a2e8cadce28f8bced", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.201.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa40a02117b0eb8d1dbe6419815ed48facb5b7f6e0bfbd0161b10f887bb862a7", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.201.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dd62f4811b9a86c8165fdb23524fc232d5b4ee828bfc89ba291fd962ce1247fd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.201.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e99cbe435eabb225ce9ff566ab46660643ef52f6ce75e27eed2a76cfedb10dec", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.235.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a1bada705e604437d5becdbbfe77079dc6fd62f98e3671a2f4b18ea22e3099fa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.235.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "91f3840789c0bcd2f7fe11af30c6a4231890d117c7be52348e78f2be66dfa70f", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.235.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "72ece058ea830140981269fbd7c82f5a8c05fc0f8adebde4a5e44018bb9d0b6a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.235.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "589c0411368e2110a5f0d75176ea6ba6d301801377d1b5b81926f11b985ac616", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.235.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1dc7c03df83f6b73843388585afc8f5bd031624cb9b62f773d16ee9cc0a678fd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.235.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "55b7f4f6a9eb0bd5e1275d734d854c3665fe22318674ebf4ab9e5853b514646d", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.256.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5aec7a08496a22ffd3913cc125499d716bc75dff1c8face2dd95802a11fbdb2d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.256.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "82037901fc24757b735c7dcaa8ffc0b3206e305646b99b68c557eeb44399c02c", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.256.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0995ea361eceedf3eaa27866b7d06f4cc442f0d651f37d82b18388e888149069", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.256.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0fcd68e70ec98e4f9b0fad313a4d4512d90b147c71c6c74b05b9e4e0beceac9f", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.256.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "587307516aa73d710597c01d28acc46b2927f607f467448c1a54b97bf34e4e54", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.256.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9daed2d9e877f0a3b27dbd24588a3edd330e0269cd49581e3a338173e9386a96", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.277.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "757040aa3f1aedfbc311c239feaa12a828805a221a0967314cb7695e1106a52a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.277.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8df0db861eb7beab7c2e7f28cb50640b4596ba8cfe3ad551a8d736abf84da3bc", + "shape": [ + 32, + 16 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.277.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1a621b76137dc64b86663946a8b786d6cbc6a0fa2a24044307a4dbd3a3ae7a17", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.277.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "25aed415dec5a003f9f80061287fb0d573f4e720bfb95f5134a6aa54a5c8668a", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.277.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ad14ada47c61a84b77d78d1d53e39daf10b5b93de11d28a99e5c28e333297d47", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.277.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd545f24efcb6d0e6b5b4bfb2ef7d5a7f2f07c8a1f0e04e392b7163f58f3503c", + "shape": [ + 16, + 32 + ], + "shard": "model-00048-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.5.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cd827dd7e227d88d00b6a08f051c7bb43eb228a589cc043330083a47de13e2c1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.5.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0c0721b80912d8b09c333a409ee1bbed849e5fc07281d0123a8cf4ac762e351a", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.5.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5c3d656c5bc00e1faa76fd45e63b535132d4317f52d24098e66de1661698467a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.5.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b8b772d3718c4b3d0aecf0b0cd6319a662008dac613e910b137c9860378b5096", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.5.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b3c8737624c287ce3782ad9064a561646a7f1cef311488bcece4ec54eb6fb97a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.5.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6f4fc97fd538cb75a2e98908f74b1f8feb5fdd4baaf4819e20cb40b98a2f5414", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.62.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e581e4878dbc96278fcaf1ab5ac342be343112bf5960c49ad503ae2d3292616", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.62.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "da0192f64fa36f724cb99f6b4cf9db821ad221a22ce755745d174514aabb127b", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.62.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d34b8d6f067d01fe410f7624e88611bf44a0b4785fd2c832ff3c045f38e1ef0e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.62.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "50f075520554c36c69fef52bf909e672949869035d5201bc669aa4f31c42e732", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.62.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6a6373a079e03a2e39db3e859484157fab965841934d38c5001d1aefbf38bd9a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.62.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ed815c07f17e1b69f29d845aa9fc2f466b1bc81776bf5352dca2c147a3f42291", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.78.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c80f7d3933008d6109ce0b11ccb41d2786660906edc66169fe45efbe4b1a975", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.78.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "07e3dd106e4be96c54971ef071e563111b17109f3737b90bcbd927969d057ccc", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.78.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6000b936927bfc5f61de28cbde1374669ffe9af96c659b777dff7471451c2f29", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.78.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ecc79898778d29d0c82eee53c2cb9df5f91ab1be03af4299a25e4fb0e35a5270", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.78.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "db1d5b64e867e074d8bd9d549842094c962eb2cc6de725d4e6afd285c694d28f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.78.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9bccd8dd46b50d1fb69525fbbe80a961a9f6312b670c5b857609037b6cb40503", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.9.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ac290bf379fb6789f941c25797394ea8378a4da7a90313684843dfa96b19f9c2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.9.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d8fd811cef12a79373eacfce849696d3f9c48aa900af1b3ebeadc866108798b0", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.9.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7f5d3c4c5bf97a0d63365fd8cbb2b2e6460fc0bfb6b6687dfaca8ad2bcc8b11a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.9.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7a915898e591995d4f7b6dc1e850ed3a6b52e9d3a9f6db407305b3380761c14a", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.experts.9.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "88960a11c6e0dabb6d93a6a49915ec6baa56d0da548e50abf74f51c07f845fd1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.40.mlp.experts.9.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7d39c5ba44b63551ddb38b0c87e6d9b13a38574b5de2c9c885d2ef832565a9fd", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.40.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "68efc48c22648b6b0574cf557cb63bc60bdac121b8eff0b4eb5313dfdbfc9f64", + "shape": [ + 288 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.40.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "afbfed12d4d9637420ba0d451eb6f76a6294bd5e52fcceabfa088cf22c781d48", + "shape": [ + 288, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.41.mlp.experts.123.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "55431324ed6be2b0fbb2a4dfcfeb36b238a36e77d8e6e442c44973fbb23d0595", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.123.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "af31ceb3f535236000c269ec136ca16f1c75a093b9c94795fd13d59b541ad0aa", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.123.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f9a813a51822af1f89edee3884f8eddfb950179b9d019289df215a1089543b39", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.123.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ed5947a2128f538c3a911337cf5408b2fd21829c9284f8a42a71adbb170d8d31", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.123.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c8fef9463b21c14d0e99689553073a94acd39539005b792773248a10e4ef006", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.123.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "398b1982720370baba344109c5c6c3b1bb7cce2aadc95287f8f3037ce3c749f6", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.125.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ecfbb481bbca093d3be3f404f4e0b203fab1077d1e34e48057cf487317c3c4ef", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.125.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3bb832b93386a56e1c64e69da4c29778d4e2336a7c330e536c76b87014bd4700", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.125.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7943e88058716eb57af6128581ae2355e8eca038f10bbe4bf0c257c9a4641901", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.125.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3e16115359f9058896ab8d5e30488facf29602849b582e0e9cb2197d18619cdb", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.125.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b75fc093cd6748fe1b8f48871686072ebc29716f24ec49d19dc6d76010068b4f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.125.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6348ce86de7f3cd2760e5340c5220a493311cbde1e4ba37562078a7b041a8d37", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.134.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7c7ca49f34bf25c47e9b7125f115e13d6ea51f5ca29ee3f6e02ed00fcbf57209", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.134.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "043968a436e789c6fa1d78efcf5e4a4fd5e51c6158f4c3ad462f0461b9400b38", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.134.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e4702008a6bb4538d4d1c520be42022d1a22c56a25cdb6d58d83353396008c97", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.134.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d408b22f590d56a08d9d716f34c491a87ff6aa37c214080c0b6dc6197bf8acaf", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.134.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1860530ae693734e0b9b6d8f3bf66d8cfaab8b9c33c6b653014126dd7e388613", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.134.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ca636680b09c210cca2c7f1baa96b9add8eb575039a0c712ed766751bc21f7d", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.141.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f7fa8d4c05b80ca25a7b1d12813a671b9c9051e78dad2fbd06c0431cc2d019f6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.141.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "521853a3674d964fe73fcb24f55be4f958f17f1848af8d1a9cde41f00db26dd7", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.141.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "55a8ee57842b193dc0b88910164848f77bb5fc86cdc494996e1e80edad8b8001", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.141.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "58cf38ea4031b980455568175f01a38e4c784ad6b37b0d818437dd0b5f629615", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.141.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c33a5dacc55a9812d6e42c8609369f4fd70ed9f34edab7581c4464ff0488148e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.141.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9bb64850e68341944409fdedd7d5aaa62bc825197d9eacd0a2fbd9f30f7543d1", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.144.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8195c3874834bb21b7396042964660511a00ea45e4a8b073a53f95c6ef467ecb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.144.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ecb4860b9c57e4409f6532602d174de0a2bc5fb575386913b5ce684eb2e4052c", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.144.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c8cafa4bce7b39d3e573066bae46d1024cb0203ec806d78cb96fb03902856ce8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.144.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01fc2d875f70cd06f5a12ef6ddf888f71e3d478747c5e5222c5806f2f5b23a04", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.144.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bf519534e77b44c7d9c5f0dfb5246570aa39e78d3efac8ab68e02612900101f5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.144.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "648a1ffff0f9d9fd9bfe961e897cb20c9796c81e55c054790440eaad6c6107f2", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.154.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "53a0077c7579e6430c56580c36417840c588dfde2943bd4f346415fa1fae251e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.154.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f08e11258dc3f60da759f0e1b956ecd730c9ce7e84641403b2d9d0dab10903a6", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.154.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7662c865a3ddaf39f1d38b82e652de06aac3a3ea945470d2eaf56be1978493c4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.154.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5935ba69f3179698aca4d2dcd136ac8bbb93c0982de9911b6a96ee0bb9c89083", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.154.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "498bd4ec6f5d0cb09cd86c54b0a80bf75269c54106e2addda7c557e7ec5e21b5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.154.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9f600190e29d5919304cd8ca67890f5986ac76ca9daa3169f1fff062a990dbd7", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.184.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "340694192fbbc666f326f546f68af12ff82ffe8d27c0a4c5c56cea4c157a1707", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.184.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "47aa40297fcd322b8f732d2f16837c485fbd221440ad534e67d495d6fe79d523", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.184.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "845537c9fcd95e5ce960c776b2dbe56558d362a664be98f6de9fb76554e12860", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.184.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c4946265db267450b07cba40be1c89ac2a616ce71c450a3a727c2286bcbe99c2", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.184.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a80acba638e061edc892fc38d0cd4a4fcabd56ee977fc185b7b02cea7bc360e9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.184.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b988138bc09bcf1b71767814a58f1f30c38fec1c2b4619cfce349c849cec268d", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.196.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "affab1ae4682f435ee9529c3f8d73afcb8f2706478c3df7068463260220c4881", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.196.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d1c7bbf574cffa2ad8fd3080a5b64254088eb1536c5f1559051107f3e9d28aff", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.196.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2e0f2e75dc3005669041e5ecb4bc67003bbb437783d9b74e3dbff5469012b21", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.196.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "48497c47ba3e799a7f6b203a4dc4735e1f6be3c36c8ad27f701fdb9fe5f528f2", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.196.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "deaf004e3ed4d92d6b8dd9fc11ce4919c6de0a8471f21b95be95d6ffdab54368", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.196.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2df786695f5fe7ff003b77fe6fcf68654cc1000170666a6f7f2e82ff43896e16", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.20.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2abe28b99b8ac307d9231ab0f46e2f109903d25d76075898e5d33131da51536b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.20.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8b7fa521e3db6111222c5c07fa399064cfba1ac18f191e85f6e9b5238eb3d3a1", + "shape": [ + 32, + 16 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.20.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "be35b43289b4a631b80b66416c242616b41663939a6e869299ce5d67230bc18b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.20.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8b209f02889a9ace111fcdd6ca5df7aac335d54233ce1124633c67a9e1e05f87", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.20.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d3ccbf5879592a381ba1651d8fffcd85351f2163aad44bef791be911f7422f1b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.20.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8d26ea56b9bc52dad6c0ca7127a4a9c0897a5477874913d5c6c4860f9211d827", + "shape": [ + 16, + 32 + ], + "shard": "model-00049-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.249.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3609fd42e437120efc565e96276df3859a4a5a9c1ddf894cc885b07447aa97df", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.249.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "52e75f167f423c2b02b2988046b0155f872f3f949aa947bcf75b3186c91e6093", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.249.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05980c9e08fac659c9a63161e2f8776c172d50bbbed1ac5d544432a92ce0ff40", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.249.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a853640336bab4c6a19bed1ca1b402f55e0a5ad4f36bbdb1767d7e737dec654", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.249.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6f80cd7e3c207cc2bdf3aec11872e1cc2b9fe9d6075caae12e4ddb1afb97b171", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.249.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f90f1266909cc4a26d0b1e4fd8dcb6c04350ddd75a7ca68d766268efb1feaa1c", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.250.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "be4f285a40ca1953da2ec9aff3cf499eb3ef044f9c96518abf63e3984abf8f19", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.250.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "87efa4ff0f58454f991ebf293a7a9bc6b662a4e4ca94bc2d52bf560763950d70", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.250.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8c05b3d5e821a2443177efffc1d47b447a5474e4cb43d09d42c6e2d17736a8d2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.250.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b5182cb5d75c4ff1e771b553590eb999ee210446d2137f1981cb1e5d119f5a55", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.250.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ea5af1ca3ffa550931363420b93c9ff6c22e058203d2f23813c550dbe8d02519", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.250.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9dd3ba82d8a37a6c45276d170027735dbcd9d1e3b23a6e1ce567cbcedab3c69e", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.34.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3f0faba6977c203bbf9f07188b15786fef2400ef06718d340fb5e1ae0979ff23", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.34.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "409e72c1c38c9a6b89729e79742dabcb8dd7728aba58465b7a48cfd603a6965d", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.34.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "97a1758469feeb66e3ee51ecfb7d87949a5782762e6ed4e296ca85be1fbc04e7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.34.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5e0a957823011b9d1b6148e33c714e27ae2cc7c929ef8f5107a7fd329ddb598b", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.34.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3c2799735140c7e74524ef6eedcabbebf56e5f7b412a5d3c853cf4d10c3b5813", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.34.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ddac88134c4760f4daf0e0510207eee8d1e32d36d38185c68e46accfc3cd55f7", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.80.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "29bc09eeef0651f0e139a7d8931407b8427192dda5b76555329091538f0e85c7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.80.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c8ce7b9140a851599b844c926f41af60b6846346aa3230dbe454bb9cbd7e8fde", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.80.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f441388808b2549138959a064c6f0f79170227589e155b94aa16f9f3e993ce0c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.80.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3d32f9f67d3612396f704321d095ee65e493dcc5fb31e7ded1fb69e94f09f480", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.80.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "83b78a430e834e5f338f7f9ee116cc37296f65ed8ff25fc11a3623e5d0ff6a52", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.80.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e5c708eeaf456f9bc1af119b7b93d8e3f9933c1fd99965e2ca7ab6e08870a399", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.83.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ffa6d9ed1e1b7b7e022acc64252a16f2ed1a7f63345ca11cfbdb389216e14c72", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.83.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ad023cddecf2e8c94ea3e0ec99a0f15dc2bfc5ed8a64061f9d30a21e3abfda5", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.83.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d625af2f6eff7b91e235ebad126261dfcca38c76c02c8d98a496e7ec400867e0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.83.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "564a1431f472ec79f28fb50986fb1d4dd34d15e49ce52ca1576f7616ee701784", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.83.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "66962522a4cafe9e0385d69d2a8f8cfe4f1b2714953e2f887d7c1180716e1234", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.83.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "040cb95443870d7974dd1ea1217d2fdbc00d54956a4ae8dbb7575ce7b6c2cede", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.92.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b4b0f9b20bc33fbcda4f52ff5d87cdcff98385db5f187f151e43833c2d221fdb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.92.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bcfba1b4735820edc8d00b3bec2fe4e855a2bfe73f3b810b41ee7af6da112f8b", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.92.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c48b924df140fc11d7b1ba951f428399dd2b84e27628d0ba87fee551cbc742c4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.92.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0bbaa88728aa498962617160473dde2bc3a325a9378d7836196f304348383fc8", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.92.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "81b7c47d3b36d3f882a18dc4a263c1d0b031289c5a0234ef137268630ff9a87a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.92.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9f346c699ba0b9601b13eb904f46e7efd385deeed6b1460ffad64fddabd21a15", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.98.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "60d90b275d29f57f39e8d03dc436d97ece0561bdc5af7940a72c0ac1067ffc28", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.98.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "54d4ced7f0cdca6e52fc91f7bde7ed45bc54c9f604a15cebfb2beca50a196e55", + "shape": [ + 32, + 16 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.98.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "26dc1c3895a988964831c90c357252df5e3fdafb37a6cd76283b20ecabd07769", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.98.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "221a0f2b7cc34b5fcedd25488643a0841fab5be3ffd8641b954464d9bda9e7a9", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.experts.98.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c5186fc806a3912db794d45841a801b75792e3afad6b9820fcf9fc8f57aac37e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.41.mlp.experts.98.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6d53e40a9c439cf64aa77045f301dbe4d9911ccac6c2f30906282feec5ded578", + "shape": [ + 16, + 32 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.41.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "fd2e2239632cb83f0be3eb407732ad61d68712fea66f9ebdfbe22a4f4ad5169e", + "shape": [ + 288 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.41.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "e8d3b5408161303ad666014289960af52e06dd09c44d23f412a5752fb1825872", + "shape": [ + 288, + 4096 + ], + "shard": "model-00050-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.43.mlp.experts.105.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e16461fe597164c4e546d7c317e5bb8c7af7d6ec5e9beace3bfdb61cb4c3a6b8", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.105.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f04c28b34b060252eab531974d478242b368408e7aad32d95835fb2eb82467c9", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.105.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "71b2e11d78730ba5535d2ca9c7494e5b6dce99e1da4c8a0c80e8e422416132c5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.105.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16c143ee51a4647ff0550e1bdb2cd77a6be24e2a0ac5f2dcbd278ef9965e0a29", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.105.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c52f8c7c1faeb964dcdf20eed9e7cf8daf2c991fde1b3c6c64c0275aed4f1e3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.105.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ac762eb773364733b4c71ef67a00eecca819d4929de83a0cf164dc3d7fec26d3", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.130.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c970b926026904e39237a66a96e111524a62953472e9a56e946a950ae90a33e2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.130.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "717cdda87230ace40b05417e64e46f423e318c955127abd09ae2aef296dd318f", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.130.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ede30b929a9c9e9fd749147221ad065f3729fa56afbae644ac75b16576322c82", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.130.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "103e3eecacce60002108b43567e07cfe37c7ca94b77d990960bc56b8449a2965", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.130.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "531be422ee5342c9ee5e9974ec7f656c0d7c77776494253d9d0a408620750e8d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.130.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "94d3e02a7782bf3a3feaec8dd21b1690cc9230c9eaeb4e638506e4775f54cd18", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.147.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f98ecab516419c0e205084be1f0dacb611c29ee77612154afbabc5133e0e8e51", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.147.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4a7382df49ecf3a05950c0d679b4c7a98f19ca9f9fa479b9742f177cad339c06", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.147.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1e060dbc2d157c2aefe14cfa8cfb257947e6baf727026ff556ead323187f0dcd", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.147.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd178a039fdfadd561051bc704ca54d57259bc742862a361837dba1b26a1217e", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.147.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7ba2af93f8f03c06d930524609c2133fc26350a4213122f2851c3e46a065a7f3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.147.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "87a3375611fd3d44f0c48e6cacd401540a01f829eef94a6f880196e6d5d6af95", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.148.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4c6b54db539dc04a59217c7f021f0484f532170e562aced77b1648e632b53a69", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.148.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b3b95fc41ced5bc174e0aedb2a27829757cf5e157b0742e63ad1f051bf60b13e", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.148.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d3288507d507bf04a912488e97661d12cccabfc41b20d3330c14a9318580f157", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.148.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ec6eb50d4b89ef30112d1b560c7f3c7f10288a6f702421361e6f14230da85d91", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.148.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3edd17f4b3bab97b9726e4d36840eaea02f96600a1fdf0c55535d8022e2b3080", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.148.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ece296cfca7c48a3760b2233d964ed6dbf263744a67f32c0a70afb119b706521", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.151.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "45552aae14e036f010a5da479560305b4657ce7651cc92c19d03abd245455857", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.151.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ccf59606ffc7a64be5dd6c03c34f9b1a9fb55e20e13b002c14313cf0bd1729df", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.151.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa6acaf3f6338fdea3655d24be61429f14231f13c05546dbae69a2368945a376", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.151.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2a6b0c3650d76dbb8920d97e5bece7b61c940cd49ba32184b1e741178058127d", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.151.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "45cc9a9e7e3f8eed8185f3f43dec5123aa91342e9ee14ef1101ad2777d9f1633", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.151.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "27470ff57167d14b2ae7fa69f4917b36ec44b8cd9bce9a30ef43e52efbf5151f", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.17.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3a85fa38951679bc18f9dcd4d96f6e6eaf9f76e31a00224b5a49cec91b0b7296", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.17.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bd82af45804c00e07dc1c0063e4b968befb2116ccf437593036f4d9370b7cc51", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.17.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0364e8c18511b80741da64acfe269886f6b1d713ba72eb798e3e5c4fb9d8674e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.17.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "db766811dd80d1518ca7b16f32d9a14a6544038ab6ba6c7b8721b2ea103c9b27", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.17.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "996bef5bf31072248339e1b774b20a38002fee45c138a0e3866c52c35414ce8f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.17.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17cebacc315552714baec50807ed335eea4f60a7db4e66a33eecafdb8d1bc04a", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.18.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "96f79e86931fae47be4d8edfedfac5c93d109f8e5f2559bf8dd6c5873c3c90d1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.18.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9d04a2ad4196d47a3d6fa0f59cb9189a6967344fc3e91ab35a4ac3f32c70013a", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.18.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5069ea0317ec57be4a4e0c31f4d657d8718868f290fa00f8fda3cd071e30d8e8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.18.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "af9372b7b2a77a9efc825e517551772ba602c523e9823228d5aec99b436fafef", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.18.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "04cd854460b7269e42c06a34f00539d784e1a66f0658d5ed35aef7595c284e93", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.18.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8800c2f804205cb56b700c9949d060c5cc654c7a2daed6819269353b4ecadfcc", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.20.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1979e117dc30f5d418852aa781935712b45270e00379d29dc8d46bedf16aac1d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.20.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8928e05d4fc2a642b6d80f04497ae8ed52eb67f72c40cc30aa38e5dadea7e6c6", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.20.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a2b7a1f2b52d0bfe0c9785a3d9e8c7c6661b1168e4e37e753b3dff95b2d46067", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.20.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ab5cbe164f0110e8cc96a34610bb8a204f1759d575e2e14fa8fbf6d88a3e45ab", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.20.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "32c418c13975bf36a26558c84faa4198ff71a2a466e91012137fd1efb132e9f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.20.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "782d2900ce376f51f9fead301f44203a33934a5146aceb929d206b72f0f1ec70", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.207.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9116050a99e3efc1b6f2662b9824c25de7e6b4475c08aef7c9be1403743ea8bd", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.207.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ab78fe37354c826620494627e6c4ed0ab83030ea184b50477e3b58e111e526d", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.207.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b089253fa97c00a255bad52c5c85e12b2f711b5e6f34e316e30ecfe0b0d8e8d7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.207.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "acc8ad06bcfac804ac696c78b459b96e7f5c0e8ce5895f32599c87ec1599306e", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.207.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1a2f9737b6a83bdfce0f20367086aaf9f629be2f9e3b4b4b7c66cec1654d6e84", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.207.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "604122024393a24244b7579b47d880576149176f04bddfec8fc8ccbe75ab06d5", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.221.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0005395f3645789789a3fda93fa8f84219fd9003f051e143bf3d11bedafbcfa1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.221.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "536c749b204db373d0c130faa140f51c9c632da6d022f145895beed00235e25f", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.221.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9b7117334ca5496d535455c59c58c1db70b99d864a866fa639961dd98f7a1c1d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.221.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fe6cc7c865fe9088ce521c2907cdb9f1b3129ebf912661c8ca7a9904ee70b0e4", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.221.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cb80507fd96e7c2c221dbda0a599af91f63275ec6621bff43ad34a675164f8b5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.221.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae7faaa597023a2af3ea2c178f697e7c24e737af6769a341b55a4ed7e94a3afa", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.239.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bd3cb04ba23d77824041e7a4b0bdb49c1d19e057553bdb8b192efa0f8591c2c1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.239.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6bdd4af185614b97f1531a078424471a9ec5605ffd10b1e9e741f0605d0b0a17", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.239.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "56d0aec6487b432d4e7e39744d83f8f1ba9039d2e15a2fce2b59cbe795f812af", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.239.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f17d12b6a92c4f732a849331ccc74c14ad8433193829f015db183710e6b00521", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.239.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "349b65c8b6ba6d4db86f18a0d1696ec974217f2b5784acc980dface16143e81d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.239.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a9146dfc315fb43eb32dadc676a334735f0328cd17ff1a002bdb4b9a2deaea2d", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.249.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "89e8d98a057e82b4656db6c4b6964513fc40588bf082b7a5fb068886fe35416f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.249.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6356d3a2d0c4563edb716818b8f07f2eb07925a6f208d4645a323367e028feee", + "shape": [ + 32, + 16 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.249.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f2b7b66b3dfac7e5767cbe9958ddc5057341369b8446396648a41af7341a0959", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.249.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f31789e42163421c119243890d14de495d8a223583ee7595170afd9b3c026928", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.249.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "52c4d472b54f1b8595c7f8834a3f4e4f38112c6e09d1f15cb58e20c2985f63a5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.249.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "368aa3efcdfc75986fec9b3d6caaa0a1231a7dfcd2a416c4f98ad5267b62bfd8", + "shape": [ + 16, + 32 + ], + "shard": "model-00052-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.35.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cedc03aa3c65eb1585d4a7542187d1c32fdde123cf5727001bf2f33ae6ddbb91", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.35.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7a8a1892a8f58c925b4ae73c2297a62f7e314090a721a5a4139b76e769ba6a7d", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.35.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d96387505437f43f6726e544309b272860a4c9319c0204b2f09d77161d85a381", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.35.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "11b11ff761b81f4eca7b9de4f79f0d3067232522eed16aac2a3d8af59752c20c", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.35.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c3918185e17cc0f3aa71c7bd1a1e7b59f6d2366e1ed7a08408f54540eb2bce61", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.35.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "42cfaf5a6755d93c45a787abdc4d9ce8f384dfdd3913eecd8c7b521086f15ab8", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.52.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "930bdc045e706db808f08a4ba42cd361b78023800dcd7d982b2f3a93283e87e7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.52.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "17a8be3961b269c75bc2fc3ef7a2185120ed41d3f2975f6b421745857430da61", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.52.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "48e4a5584ff44b6356c2263a8a67661417b0dd18ebb666f0e91266538d3c8a92", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.52.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2640cca6cf5418371fd76332c56e1df369f5db3552cfa2d51e308917ed2586c7", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.52.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2d56f3dbaa28aed48c7fed64f85559efefd037acb5f38b98b0fa4bee43437350", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.52.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ee09fbf5e9c06b3574e5d726ff12a4e9f6b065a7d87b1575df44817c9abe0d0c", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.84.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5918cca8864de6d2eea43a9861035ebc8a383781e83014de2a76bab73e37ee33", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.84.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dce3a7f1352c606618bbea9de4bf1ff844120a46732642af477f2f73dcba4023", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.84.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1c089c8128691e0f9c0c2e6225867b259fbd99b307be09fe0743bd7e985e41b3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.84.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0501744e4a4f38bf784cf201ba97a4f3cb1ec8383e04b32ec57078d942aa73ad", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.84.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1caea73dde662b2d6b27501040c893ee12bf5197041a82af4a4ba5c3b34534bc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.84.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9f06653a11a7e8665d81331051d2b2dd082323c66acb544cbf2af0d6c21666ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.87.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "31754738bddabe497eb2709ad0b6dcb9901e5b104a8b1a8861675ad1d8793dc5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.87.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2567fa59c7e661c43c06b31274d48553b5c58b2b45168a6223a2be3e0526ec2e", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.87.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d6e14c2ec323258d85184a2d3e0afebc690ff0070b195ad4c78b14f6fe02b45b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.87.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "da8589a9c8542b31b0ee22b3bd2c458f8f39f381fcd0db01908b88a4cfe43c40", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.experts.87.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5477fe0e8d6b48e6e2b25404225c4f8b53e8aff977c3943eda34a3cbdd0fdc69", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.43.mlp.experts.87.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bbb36aa5c384e38ebbe2414d0643bd7a8a1e963cbd465428de9c2a14a9e2cb58", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.43.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "32dfb973f024cdf16f3ad5cd2b5bce426960b0b459ec380335141f81a7a0319e", + "shape": [ + 288 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.43.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "efdd22711058dba43e2d12057cff64456d72246cd198b79e266a39032e3d5e3d", + "shape": [ + 288, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.44.mlp.experts.113.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a2a70295d26ea2d76853e68a6c97b7a5a2e02f496100d74ac118a8a917ecb7a2", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.113.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "facc65980554ae1cb0849b3c0a9d87287a1c53a02ff6faf5f6de6e7014e7a655", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.113.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9bcb4cd6a184ed4d2be6731e78da1e4bab36e94746b97bec76fafb71a0e92371", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.113.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ca8fc0cacfe56b7c82a076b0d03b5ddf1d387264b6f19136aff2f67c004581a", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.113.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0ae2a91348f6b8691f775074eb297cc8ff47923cd62305ab0c37e6d343761bb2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.113.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "19ad35c93c9029f9b916dff59ef824042d38a4b40708e6cd4f83c5bb0fde8737", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.127.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e92efdb50dcfe756cfffc41d9ec134f25aaf736096ea9d5825d530d2cc11e74d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.127.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ad97d2229cb5b90b5aff7567b434ebe4b6b3f78b507d45997efead78bc6e5c75", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.127.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2b72d02a93fcff72c447d1bedde9e8b97a00fd227d0c37b408bd0a67e4373c5a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.127.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "02b54eb8e43c177839fdea956eeb663262014f9a32a4744d683d0a9d4f5249e3", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.127.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a102404ca1534317f481de8af943b21968ff6310f07e73e82ec89b0b92399227", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.127.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bcae291cfd2c4242dbb0e8b3e15f1e8b554a49e5217453953920df92bf12bd8e", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.135.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b2afa944f16f291e453bfc11721aa4bdcb106471c32099f43d43d2d6645f52a3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.135.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "034e8f38fbe513429b667a08ca6a19afe672e6f337535e0af85c530a50a80005", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.135.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5b2725a1a60e3eecb7280c23b3709e39585fd8ed171d3657f72769ea33da9bae", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.135.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae369e64efa1f8a49970d1dfff375951c2540d6d6190818da72131985392bc90", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.135.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "262c050351e57f69c995122f7f8f05c4d8f1ea8656e0d44e5fdbfec6714c3272", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.135.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "913ece3b3542ad9ec63353c8dcbddfc3f7e620640ec21f5b1c34c829c7437ca6", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.138.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "411630144fc34da125aa19d8b3d95af77b3cffd9271ba3640903a74f0a8305cf", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.138.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "59a937774c30a90594d4e1d50fbfc26b78b55d76a0d6a452a6fe8c146d3afade", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.138.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fdf144e24e82b7d09109b83e692ce0d41af098db3275cfba50bf9720c49a73bf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.138.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fa8cead542ffacb6a5c9bf4a8950ae4135c46b7b80d4c8c73525f1e80fed6ebc", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.138.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dc3f58e09a74260167608c38e376a3b17ad7c4512b4ecc1d427e9182bbd2e1bb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.138.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c3bf4aa2293bbeb2c5f9c5c7623510232e49ea701224f354f76060f79e9f338d", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.147.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ebb6fa56a3d6d99201174643bee13d394123507b67314ac8a42b99eecb25fb55", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.147.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b0a64e40113a7e29a6e18fde5c1613106e62e146ecd85b5d445ffb2c355687c6", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.147.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4b3ba037ad75bfd2f749ddad56ef292b1a358a56882deaecc8c7a528c3378b6e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.147.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "89d1b239548ec4ed6e1b32658e0a55c5d7da92dd6599d5bf680939842fdce4cf", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.147.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c662e571b0abe90edcd2ad84ef69ed9e590cbdf8d18fead5e74a436dda435f0f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.147.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c5966e927b922de926b159c59ce660359964a8d9f8eb6962a3264df79a0c3c76", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.152.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "57af0c4855609030dd0faf9c35661b64119cd9bf790353e6cdb718f05278787e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.152.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc040840aa41ecab2c0ad8045857c2eb993c481ca13349a2f3f0e27c495c352e", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.152.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "219550a1e85347ebb3f5869b11765f844d5e7749e58719fd84f3d5f8193711e5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.152.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b2abb35369e3dbc7edc369899a672c774c573f8ea76b90813d69f5b28bea8a54", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.152.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "29b2dbaa544ec635d93ce97cc84bfa6f4b523909a7100439650a6a9af6c2e96b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.152.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "742cad23a2a1423dd2dd2cdac2e016577abf56e9e6350a5dffc43aec55e35438", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.156.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f4fc5dab1b357b2d3297bb117d694fd7da3d08f54258012746c714438332ecea", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.156.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dfb8f0e0e7e1a257c7fcc1831a33033e40a4d18d91ea4ddec5fb534c2a088ca3", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.156.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "edbd77a41f36528df0bbccf538576d4f54c83300c16ab7496bb17f16d48cc07d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.156.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dcd6360a240cbfc82bdd18f82908fe5b87c59901203000f0ae7c785ed41d0a9c", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.156.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "111549df4eb578636eca426083c83713b71a2a664673811e1ad0f42f2500a9ee", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.156.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "565edf55da4849466b1377fe53073e39ee99dbdd4a46185a61e3d37a6d039a97", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.17.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b00ffca8e6155d144e77d16a555b6b8d9934054acec1da341e94c93dcbaa00d4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.17.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a6d7b564e87d80993d87731b4eac5ff3f9b3515f526539f17321ac94e46cdb83", + "shape": [ + 32, + 16 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.17.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a14139545b64ec33c36fa2edb46678cafd578258f1ce6f5b38149aa831b28a18", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.17.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c300b5bc67c8c497c622ac5cc1643c869f3575ee4769d75080c111f956bd8914", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.17.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "00271966bef8d378cff100762b7eaaff122e4f2f29c08563c93e2bc3a45ce462", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.17.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f34e3f8596096b08ef3773a9d1de699db4bb6f4ae99f01103a54ead22911a17b", + "shape": [ + 16, + 32 + ], + "shard": "model-00053-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.184.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "342cc7381fd31119da4f9bea55f2e2304f15387fea97a5545198cb9325603835", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.184.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6a7a804588983dbfa1338b371b4849ad65fae850a33cfb98cb746e00139c94b8", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.184.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "56d011a9c12a12de37696b11246a185bfdbb9775fd5d34d3c7bcb3a39d480fd7", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.184.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7ff3e8267a2f0c68483557522dce52ec262b338e561c1eb9d5950db69e6c660e", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.184.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "38128eefdf58e6ef9d2a2d5cfc389ee2f134c4b900d657d4994b5899fa6087e5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.184.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d2af89d580c88995a50ac70ece69dd0f9f4d19d72d742ed7876c45c799923fe7", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.185.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0c7cb8e637f49d127665a6e70eb4d6b7cf70aff0ffc24f218a1f6219dfdef474", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.185.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d669a23d37e62f164511d196e2395f9d7de73747a1cac486b71c57840b729360", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.185.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "dad1826c5dc2579d347bf21997f29a7a69f36cbe31de8d38be7a24f3d5265930", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.185.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f2c8d13f14ee8f3e747c31201fe37fd868b4f656a9a57c6812d0a261655941bd", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.185.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8b9de11d9f37686b877a769153fe5a48e360a88792291bb566585b3140d7c74a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.185.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1507e28dbeb9938942722d58277f922032b572d1af3b0f46c79ebd8ec56983e9", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.186.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e06a1c2c9f4231de0f33bfd6a5848bda9c7bf664295e2c102c0b75c14ce754e5", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.186.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "56e0efa6add97437ae09ed9d8c93ac3cb94e434f0f5cdd4d53a07d73ba42f227", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.186.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "37dd07943d566063ef646bbfda59fbfab60ccb0bbb1bbad8b23b04f1ef06a403", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.186.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a0f3d600d88dc81526701ee801824024cc2b2f5e125adf38a52e4de500add432", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.186.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e6e3f5e4d8b15b5018c8851170a70a94c9e6bde414b063348984b84f95e275f5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.186.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0d37ac91b65c67902308cefa25bc744e3e79b84b873372cc8cf4ae9770d9e1ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.212.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c3e98635d748c613acb78275c0e6071bb2fa92de27c21b658dd896c11b5603d3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.212.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "577eb17d3e45e99e9bd15bb78de0d934796cc5588518706ecf06fb512641e394", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.212.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "39ac34791529f54513c4c2f0229cd910aeb368f76b22f87803536cca8bcfa217", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.212.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5fa8058b8ace064bac99a5e23ca3d571e8b386b613063930ee1b74d1a27bafd3", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.212.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "13d570af7e61ed349a8f043656461b3f2e59727f73af73d8a714445ed28b1191", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.212.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5cad6fc0831ee2c9e66a9b2c32db58aa26ab2f2119e7c00666e9b5742d62ff47", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.24.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a69449702e849e3f44b15414e30792a5ac0429af609981397cd2e854bbe18265", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.24.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "de9252d0418f4ac3deb4243400fa3f4eca09ca457737c59bd85724a59e93951c", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.24.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "54241977172ae3bdfdcfff7daf7e31d2202c949f56d3901ee23a2c1b634749e1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.24.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14514a83b027597dd55c1430f994173be07e2b631e339966c31e224e6dfac1e4", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.24.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "78f1aabcfa599e16833075c51a461e5ae8a2fb44866cd26d98f6d2c41c97f1b5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.24.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d46f7af3ba365fd06155617aad55c092284b1ee959d744e262bf41f367d27030", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.244.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3e061274f446951a1b43146ab632efd2c5809c85553ea6dc059f42079ab4b95b", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.244.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2ee70d2f9e41089f45889446d8ba1c8b2995546324b85a615a1c2e145390b26a", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.244.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d6c84a9b791ed914b31f82da1fc205686dcf313441b0bc66a85e3ed6cb3d48de", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.244.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2854b52c4079e0ba71399c92e75e7d19c9bbcde8efd738207b88e3b39eceb03e", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.244.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fd2bc962c5bd6d266db46acc60f536677bb262f242988593e055a7037ce0875a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.244.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b9dfb79891370f548d39386965fe682c0f8cbdc910e3d3e633c38af04b24c03", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.266.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05e380f840da4e7c4add7fe3fff89fbe90733303f7b15b29f9f112068430231f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.266.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ab3b4f801f437926d38468f1a7451ab658c3b0968682e4e6b88757ffe562f6df", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.266.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2bb938edb8dbf5618b50b31d6e3be92a2fa1e1d8ab45975c907fac0b6c2b7e3e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.266.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9028c0bf25d6d565a2552196a8868d22d2b237106deb08977d2e3338689ff55c", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.266.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "62fbf021feaeb47111c90d2e8ba4e446def249fde2cf66ad147c8be9c57d6179", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.266.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "06bf4a9f05ff0144278c84be66a6f1d6ce93e75968f627e7d616b1a20aae64db", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.44.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3808b889488f064784b92ad70ec131f1fffc9b4c7e8bb1a93a1c1c4353495fdb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.44.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1f261616bffd1fece114d9999a96a71ae4f96061f141702a72be2f77c9d11c51", + "shape": [ + 32, + 16 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.44.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "296e85f9fa3084bd086f448c7fe3bd831642fa5b6536647de76f5a9eb210d637", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.44.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "58763c932743949fabbc81849107f77c01df166e98bf6a64a2764bc0ccbef28d", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.experts.44.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7cea8c3cff2fb4a50589bf65e81364de0058f7c09f8f21244a0717a987721bfa", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.44.mlp.experts.44.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0ef2be041e882df8b4a42770bc18e9668e4ad62d0092da2cbc87b73da237cff3", + "shape": [ + 16, + 32 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.44.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "a1b0b06d4baae362c0d1e616880b8b3957aa3b860cb1883c6aceef452f694347", + "shape": [ + 288 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.44.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "ac3ab29162d27b965d6ebe42fc44dfe8cad474517167b7b322950d68882dd631", + "shape": [ + 288, + 4096 + ], + "shard": "model-00054-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.6.mlp.experts.104.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e58c19508b583404826059681ab31f4883b0f9ecd6a2fcee96a5620e67d1dfdc", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.104.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e7b5d45d8c99c5e97b7d6a738653fdea634db6ef7a90eb5f7e6744157756d0ea", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.104.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "eb9f60dc12a5fd1db22a4c9bf02cd0d62dc6fa9eca676b0cc1e519d26580975a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.104.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3dc0090276d79f8913c84a9f8ba9433e138cdabc1b1084ba9b9a926e0cd95be0", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.104.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3d122083b4e320602f8b6051de352018dc03c67c448ba3c2a24181d85d7d30ab", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.104.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3c52ffa3fe79515379d70d3aa3540e5cabad991662d530c03868d34eb392b5ee", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.137.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b3644c9a96f331d2771efd3a6bd0297b67aa156c9d7950056665ba7abdbfb4aa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.137.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8ee2fe31156236c038cd7ee6dee16dfe609babba4022cb80416bed3f7ec2b8f0", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.137.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fbe4272578dba31b29c89342669a5da44e3e940f15a9ca52334508c5306524ad", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.137.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "13570860c84d2b165c13cf1b56f7bfad0c25269bb7c1325d0dd29a6a201abe0a", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.137.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6ae6e520c2413dc62635ef9c6b3692fd9d8076586aaf49077259c433ea623aa6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.137.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "60e348d0096cc1e9f4155e02ae1ca13096b38454f9ad9de00e5136ad8da83d2f", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.16.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "da2e280c777597fbe329fc50ea1877505b08fea41992cd15a6139e78c5e2bad6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.16.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "279b202d617a2035bb231561665c868f41055949482753d936858c7ac7a377d0", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.16.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "bca2a9a9f06436f9d0bd88a8031866aceb158f58833bc46b65a2374eae29b323", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.16.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f10525779d54c838cab442bb1a087b31b18aa118604c8f61603b20cd3f8c1300", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.16.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ff5960a2daf99fc0e209ea772562549ef389505515d49a5b5726f45a3629f406", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.16.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "253bfda6f5f3d1056ddc118e7bc21bf3380699dc372a12a4df8e0de46f8a4e8e", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.160.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ea89a245a181d5009f67956c8df85d65653cf1d7fdea7fc07488051b1636013d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.160.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ccc31e9415bdfc016fa164a64160a5eff5f98e61c9008443a8afce08467fd1ea", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.160.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "41b59639b3bde1085acc547dd527a0b712e3d09ab221e5efb8944048812a94ee", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.160.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "534189900ca7798babb13aefbd38496e718c54d19c61a7166c6f098f071495e8", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.160.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "22fa8ee6311ef05eeb0dc472affe219694c88095565658c589ac825a1641e68f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.160.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "23505baa25e55bb6eda3ce2a00b6e4e8d94a21a084c2f48fcdc7cd30c55c4895", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.174.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e2fda8a017becc9abb372bfcf4266770277e2042198f5b63d9143818b7b9dfab", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.174.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "40ec3b6f2513b4fcc9df536061d7dd8c896c2fe62ad6253c5d8ea96938ab97b9", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.174.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3e59dc35157048cef24f242d320168b6323ee1451532c03271f0a7b69c26d600", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.174.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f8540b48bd02469b633991455bae078e363add6d4056ba0082dc96d6849aba60", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.174.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6d0ff4226e53fd274506cbb13a19e05f7dbe21ccdd0c2e1529a7e74ccdb33893", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.174.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "16dffc4d48a9c48022ee6c1173ebabb5efadb56d0a2f62ec4b966ef9158ae8eb", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.206.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "53820f47b9d5ed232c50810b60f4e6090d6db4a2484fa61b63e145c2f0a7c8e9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.206.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d544b3cfcf513f48d45b05238eb650678ca41f106ca2d9e8a88e71e7324d1faa", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.206.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "52906c9fdf471f822f9a20e5b8948014d184404124a24cd9e6270f6c08a75100", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.206.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bb5ab9707233d3363ec4bc560ff089529a9ef87cd948ded4409edb06e3128544", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.206.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5bdca7c1efde1dd34a96553c3d412492677f4dc81979c8ed1546af67b1e8a502", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.206.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "92ba08cc66752b0911ac4367a82c31400408d00b9531af76b04fbae7d4eedce9", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.209.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d4141fbc2c77bfb4f39d938c71507745c9082592e862b337377ed4e113fceddf", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.209.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c4880672d413de0576c129c76a25f9871411b34d8ce911c8b0e89e1826abb5b", + "shape": [ + 32, + 16 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.209.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4d6e448b55f152dc4211f4c5130d58742618b7c3955b8bb8ecebafe132eb8c3f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.209.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "355f138bc04188cc697a4f29e3aecab133c1de186ba5e936820efd54fff5d85a", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.209.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5b80ff04ebb9bf47e5d1e27a380c9693db4c24f55aebeeb3070613257f9f6485", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.209.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "75bade5695ce3d9965b3ccca2f4367d890a7beea4df94fcfd761d04180c1653d", + "shape": [ + 16, + 32 + ], + "shard": "model-00056-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.234.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "087cf48d76265ef59efa11eab69a7bf7f5a62a6079a9fae6fdcd805600afd3c9", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.234.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ae4623d1ad9147bca4d9efe756c1ed9a28fa27cd70d200c8b2068a4622a938a", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.234.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "36e63d1826b347ec4ef43be65c97ec8c7af9b0062bbe5d4bc4a09b5ebb10231d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.234.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f0a4a7aa2b291ddb8bd251f8a3b59e771c165765a2fb3ecdcb2b016de179945a", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.234.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3511bbea95777feb0380dc9f49bce1c2089e2429c2626d4f5438e6541c5986fb", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.234.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "05c1e17633b3e165243359847cddbacb65fbe149b480f797c759bdd458b8c5cb", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.236.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c403cfe435b9bb0c5e78ff5dc8618f7d1755f92e8e7bfd6809935db120087813", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.236.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e3381b905a02be6716ab5aa5e0ad464e5464acf66f5a02311322a5a79c605edd", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.236.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "84437856b317b982f860a5077cd128efadeb71b5224645aabe441dfeba907474", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.236.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "01443e030dac3c15f4196869d5410c690348166dfda3d9613257aed9dfe9db12", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.236.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8f36ce30fc526cd5c38d3464b6791489ebe257528e5aab87695e090a38a39e3f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.236.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b83c5e4e20d02228042f4b73b954dff3a2dfc8d223a3816ae34dc343ff1aa9bd", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.256.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "37adffa85514ffe047e79be1eb821e3a66ee8a9ef1cbb07c22b9b305317454fa", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.256.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cb03ed5e7fee84b382519210f0c27c5de6d67115226e0a50a0054cf54c375416", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.256.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9e093e10ffe1ea9a9ef3864fb0edd930bda04052ee661e00b2013c2b97a66b21", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.256.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2fd51a7243ec7ddab55cfc81fde360ea1fbb919b48affc0fb1e3e2f03f1c8c35", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.256.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d02ab6d4e7502ccefa84cb4ae4d9e1b702a2138b15c33b78c325a51ea6b56a48", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.256.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "26c40834b2155b984e76a2ed23cf4e47403cf17fe9d1eb9d05ed27c477192d17", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.260.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0a5f8e71737510269dc3924b7ae0b5ee37e7b59b421c999ed6d2fe3299f2f4eb", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.260.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3ffeb56a37d5cb501ccd20132c2e30086192e1b898fa3d22907329037210e626", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.260.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2b12de8bcfcfd85f12bf7a1e6b3a16827f628f24825e4294ea390ab461ee9ab5", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.260.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a00c26284116c5ea100bd63f54b2f282d355fa77a3b86b696bde37cc28dadc6f", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.260.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4bdee1480c6dd0d0d1f097eadce43b743ef3df67d1caaa5b2a3f9ebddcb4e6af", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.260.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1784170635ad9871c0641049b1b5f008b8acf1a3f5b4a75e5599ee459a04b191", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.270.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0f4a5d988e3741bd90b44d4af6172725b73b5705daa0300417b4594378b7af15", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.270.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b9a5bee618242a9afdd4cad948e763c984a4179f17e359f792a44de877e07124", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.270.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3f9fa967d1a7ac187338bd4f5213ac6fc2de44ca61fca4ecb495af9d9845d1d0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.270.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "593b4d7264221fe62edb0736890523bab775cb0ee0b20880f59d40d97f71655e", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.270.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8818662f31936c2a6deb786815ac161dc208f8acc9caea806becfe809a4a554", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.270.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c02c6dbc76a2902144339d1beb59e47589f79aedbb967fe90fc7194690f3ad27", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.276.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3a1a11357fa0284ba45b713f1c318582a5aa73bca8ce841d42e85e677f615821", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.276.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2759021f55253c840a1238ebbc09464808a15a749932daa9cb3360e0df8adb80", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.276.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f71d5dc85f6f23f4c4839b671329bd3d5a135d1082cdffd400d960ab6b6dc550", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.276.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "501327ef955568549b0314e474fac3705b850842f63f6e1277d5fe8269c91e9a", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.276.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa90df828a2bb2e83535de9374d32fc2de286bbf8733d96e555bfb5b53346ce3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.276.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6469199855b3eb392e3615ed79f0611dae4a6d9fa9b4e648ee4d9b2470c9ea08", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.282.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cceda7360ab5876c611259d2241d89dd469decd7c6ddc3c1a4017115d2f3011d", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.282.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a8dcb74f0acbf159a7808d3341ec7e38cc40b3660a906a732304fde67879558b", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.282.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "152eef1f6640ae443d838aab4a790c4665da78805503d15331e983b1a9512b58", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.282.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cdecf39667d19a5d25d9cd05ff3db16ff03c3ae01364a024db026423a011d195", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.282.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8e26b7d0ce63fa3a936dadcaaaac9c37df60a3d95805c4740d9fc4af93d08f52", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.282.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ae414ab5d7985351b4d6fa0e1bf3d9ea4189f400456c738e70853db479719039", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.55.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5d33f0d7f4bd6a69242c004b9929785ff1a084ce98244b38630cc21e8fa090d3", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.55.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5a68128d94a9a101da7fee316a6ffe5a724b24a0fb2665f777c0cf28974464cd", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.55.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "87b3d2b51473df4cee713197bf3797f819165913d71413e3b0eaca3959d64b3c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.55.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eaf813cecdbaeaa6bf68803910294adf320e1b49a35ce1b30c0150545d3243d8", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.55.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d75365f6fce180c37ed3c723297b69a0c0593906450914a8210d3f0e36401ab6", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.55.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e340dbd31bae5bc2f7635494c98c16d2c9fe9120419229ca748a750f0a2921dd", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.83.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "afea604d069b8e07c1a38485bdbfdf6d9804087a15cc2ebf24e1dc1e6f8185e4", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.83.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7c27c6c30f11fdfb3fae7edd42e0cee14aac649bf3d5420b3dbfbb5876f2b1ad", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.83.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aa72f3eb9a54220551c170b6465273207af7e460889df5b19d1597e12302496d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.83.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "06a93a1924aa1747200d94e477835e42d2c48f6ce12f29aadd96ff54cb5e5dae", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.experts.83.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ffc6166c14aec242f0471342e3262d42fc4f06116e7f4e8f6b3177c87c941392", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.6.mlp.experts.83.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "57246b8ad1f3ebac55d25f150fcb8db3a0fa48bfeb03f9e0ac6bcec48752a8a1", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.6.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "392987f094a9b3caac2331a7a97c6bb3f3b65fdad5355fcaf61bcb16c91f4b75", + "shape": [ + 288 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.6.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "b9dd09f4cc27a2fbb95a225e3dbc5cd6fde2c2f7a7623d36f7919ce531002097", + "shape": [ + 288, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.7.mlp.experts.129.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "65b5511053656faf1b67a92c30a9d80d6658b8a48d4ce93922a0d088dd9f32a7", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.129.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "41a60c8a5d5e56c91c1d492fed68bcb5f210c29d721ccc7d8df925b2e094c39c", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.129.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7076f0477d9a7084093e19b1ee6d77bc2352b61b04cc971d90f358b8ba1ce768", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.129.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c7bb0970de0e99b4deeb619f60b10d970d1a595b4d32f1cffb14e1897b2c5478", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.129.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "60c140ead5282c473d7b484181e567a6abe9774b0770cd9969c588b8328a3e6f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.129.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cb258221027bf7bcc3d15d2b470ba0e8371960ca2380082c3e5106cd108b0810", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.131.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5f25c687568f8ee632d581ade4ce2c0315aaf9a2e03aeeda63edf52b8c480d45", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.131.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e304427645d97a0f0fb7c6509ee0ca319c2dce6bc5723e4b8897764d29c29bce", + "shape": [ + 32, + 16 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.131.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "16e6e077c729e51eda23e85f094af1f98f68c2e0b18761811d606a932a23427c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.131.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6b7682785f5ea1c5e1a992e95185bbfe12b1632721d8c8ead2654889ca72541e", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.131.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1b123d84d188bf6ab0032e61e9cc9659a6e56b4e1f52f343ab042dc9acad52ee", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.131.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "516190329212315e99deed384c1f8c1d7a9b412c9dd452b201295d2c98db8c9b", + "shape": [ + 16, + 32 + ], + "shard": "model-00057-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.145.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "96b3909c4ad4534d44aeaebf6c486377e33d16bb7dc137d8b6bc2adc5fa7bd65", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.145.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cf5c97e076aea860e31326610d66dec31242df6de8c6a38db63f7313c5b5ac2d", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.145.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "77c0be984c14437c1e2f9c873881d8bb86109d6376ccf8301cee0fcd697c72ab", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.145.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "5c92a851250061e7b821eb18ee2b467addc815202110fa8b550b3bc74f2f186c", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.145.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d67e7dc980c39d008f5c6b056b660a28fba5e957e6e1085d1112faf7284ef6e0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.145.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0a33cc4dde04867c7df3b6c21d8467d052fd98eeaad3c63852c972721fe41487", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.184.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c2a082dd125ed426f4c341f808da77db185a8b5697af6ef73811c33fe4dbc53c", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.184.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "609477946aaa750fa601cfae050537ff0b0af950e397be1a862e07f80a07d0a9", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.184.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "baeba319d770a815591cd107e59c02a93e0415bbf24d5f36bb7513c00558ce52", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.184.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2c2a40a60eb10a08646e789d82bb2dc8acea26b0b735f987b24b8e75a2477061", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.184.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f334baff7085db4478c36f3ee84d5c4e01aa39777a8c0d7c444383b8b1d4445d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.184.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "eb58a34f6d1c994eb07590aab3b3777a500d7df39b4648a6f1a52b17d03557bb", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.208.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d7d997c6348c06afb8c6b51d60e8ade579a8f37f24f9544c2142913fbb8a1921", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.208.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4eb881df0338a9e541fa62a34f36ec95060d095b4f101f88701103c250bfb74f", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.208.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "90bb6514ff7e52cecdc6186d525c806c75f3546ae2f8b24cb50554dfde1b3757", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.208.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4ce791b5bd101d54e843b2678d525844d9be9661b3eb290c42be6d7944ce0529", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.208.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "05e229b465ad00c0add1dd460c37d3a274d6821aed5a630ef4ef9c000c24a9dc", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.208.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "41b8943ff884270789cadafb41359037ebf9a2495267a13c9160f38d35d6344c", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.211.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1ae53590c10afab310200c12b8c7f15efdd5d12661fdf85c3010e781acc65e49", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.211.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d0611b6b9370531e419e46ed796ffe39e7b66e53921f0091659895d20c7f1b71", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.211.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "380e91be1550c0f5af5d032cf77f46869065f6fe3894205b686a0f2909135dd8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.211.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "735b2e5bdf5eab2f775ba42294489a3fa079075683463e3c15dd30cbbbe7ec4f", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.211.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fddd658ab47ff11681674aeeb1bcb07730c9d0df6f48443c6116a2aa234a1092", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.211.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "182450f941de6ce0ce1ee29c663cd2fc6ae47bb68969a1a49492e80e3362d77b", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.233.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "99a23c61dc05e5f653920996631d5516aa4eb2f2ea33783e7275d3ff46722e20", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.233.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2a3e7cea9b350325d5009871744b97362c1c6af129244ba20307842b4c13b730", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.233.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "480401de1dcc5e365a456032d6a4bc1677dcaa5133d47c8b7e74a818477e61f2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.233.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7fe1791181a7acff18cf2349305220f05fd111bb6f307630def98cb14abfb7d7", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.233.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "070799903690ef077a245671932f99227830fd18ca263b443a1109b9a4c39200", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.233.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3d94f1ca70bce8449ac588b28cd0e697f910525b6d46b7b459048810d06f2a6b", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.262.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "0cbb65370719093d976725c45df1d8995b972ece9620cc7595fd9017eccbeb97", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.262.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bc8ffde571e402a02a4dfdf270c0c7f9fd4ebd21ae24bee92b87a896b73645a4", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.262.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f76faf46916b2815c62a66fcdbd69247f8779fd2b69de349761e6cc3cefad938", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.262.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1ec41ad5fccbc9d9b3c725982f1bfa60349e0cf479d79d627bc95e9938ac9e70", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.262.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "2582704a8cd223a641c15ae4ce0895b7cb9e6601a664c332da5e1b73bdd1900d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.262.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "29c1e89a8aa92203f0f4a9650609ee6a983823cd27642e817e26687c44df35f2", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.44.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "13adef8fa990a1ca327bc368e9ce40f443242b095d3d63f1a2c7cdff3198e773", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.44.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "38658420df071f1666c3877f129ea0e73df05e993bff84576a237731b4d363dd", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.44.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "38db0f029cb53cdd322b86c0d4f0ce80d6f46d196525f5a7862d14927479f0f4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.44.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "93ee144c6b4166ff8a5a6a0ba1d667a72c36accc60c63c7ec9c6a40967dcb3de", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.44.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "141078949b8c6ba173a254663d29676db3261c7407b3f91bede31b795461abc8", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.44.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d4c9f475d881318db87f1a8d290f5fc477d01c3fe4d718a75aa9181a7d7dce5e", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.48.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fbb506cd9eb1b3e12b3f6482aeb601b829e4b45cc59f1e8538d90c02e02220a1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.48.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2d52ab458c780b6d74ba06a57670724c07ab08740dc59f22a1330c4a4d1a6157", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.48.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "c35a2101f52414b4f5cf6b8313d167d03e040af1e771a8a4e26b5ea5dbbb4792", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.48.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "539233797088ee69c3f63b1d91b25b4fd003816466278064640f1d47ca66d762", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.48.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "71fed81f5690994b7c496decdd0bb60224aeaa665f2510d920edce26fb9e904f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.48.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "fa44c0624be0354ecc8d534d9eb13fa8ff2d488deb8ac1c62b0bd574b4decbac", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.51.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6dcfd7e5ece0723ef972cc799b7726fd89f0b70485d00156d77d9c01353b6d00", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.51.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "570b601950496491a3c6323253927e5906178b11e52f23f543ad23b8a3afd2af", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.51.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ce217b2a30ead72da30f159f98d67ff27b7a8dc82493863edda8ca4dc723c63", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.51.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3862a65d85094f1ca01c2dcd6d707141ab8a0a3d4b4bf781582324919f206ee3", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.51.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "aef309586ac33cd5cc98899d47489614bc3a31f8e72bc6bb5440d071ced4ece9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.51.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9c6e51f1782d780a621ca180793c09cfaed71c082c6d4c4431a005f7b8df9337", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.57.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b8fe2655a4cd06681d4a6c4af931c1bcb72b08274a889dda1f642e1bd013fa65", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.57.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8957f6aa46b4e4285b4d6628a43d3bd7639e99d6a94fc9efc2ba9ef6b8ef1076", + "shape": [ + 32, + 16 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.57.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ed0952919ed57b27cbba2d40fbcf2fdc62d457e36b76b8c2b86fa37b7e59b80", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.57.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "bdecb1f6bbfb384ce97e8f7ea7c8f3269c9069a8bdeffd401c5e51d05bd3b3c4", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.57.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7ecea4ce7bc3ed25593e863f9bf904f27c56d235122fffdb46e9a263d6be086a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.57.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a21d2d3cea920ca5d1296feab090a9af0116e18bf363cf77aeddccce55589c67", + "shape": [ + 16, + 32 + ], + "shard": "model-00058-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.68.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f843fe52166f3a0705a173936013ccf31862a69f800568e1f3fc3706145b6e69", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.68.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "889dd09084bc045e777679503b97ffbd3d3c08620c51d32b70664595583dd584", + "shape": [ + 32, + 16 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.68.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a01230c54ae0eaf313957cbc98df8cf40ad88cc5c1bff775dee4ff8aa88cba59", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.68.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a902d9f2a02491f0f59b53cc2e4c9f94fe59d16d04d5feb7062373daa7abfccc", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.68.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "63f0d6b82b9481da8d618cbc9c4c1737fc558938aad69865ffac8193454055b3", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.68.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aa1a3b2adcf208a2bb734b3f587ca6c891330925f8fd810f8d12534bbb2f73ff", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.72.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "56f6e7ce93a3b394cbcfd254b47aef267fc51f2537215d4e1e9173134398240a", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.72.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "cee4bea266026c8d82ff8c40a11167ea7a416a56c029e868590c36203c10bfa3", + "shape": [ + 32, + 16 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.72.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "07d1988c2f8de5a2c4cf74abe49818f0ec02e14cc64bb3a1c149a7d63a6d2727", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.72.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "b3eb554d4808dc09497ca728eaaf2dc1c1d29d47f56a925b6062d4e1707ad7d1", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.72.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "901409398220d08a8cc35f7b8e267694862e598f49a718c07ea333bd52e8e6a1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.72.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a75a7abcf8d7ca354b13c3d6922936cd2fb5481824d8b3af7f917329e493f117", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.81.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "62ad8afaef8387a3bb1b214ebd0531432ad162d9f380e4a7fca11af2e876b396", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.81.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7fa2f2a371d87c82191f8e68824dee179962dd8ab234bcf7c4fa87dac75c3810", + "shape": [ + 32, + 16 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.81.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e7af60336f3c07e3969f1874b2e2909c0eb1d231d37368d89ebd54182d84366d", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.81.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "edbc432b260baa333b0beabb519864841fbea6ecab82180b95406111deff5f54", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.81.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "78df286a62f025c055d719a010232848e6aa940722c56384552301921cd68d16", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.81.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8c7b255f798a63fffd8294e8d3a886e4af3766b5a59f6e16063728d63ea52c1f", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.94.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "85594bd9f056d069484a4d48deb9c1fd783f3fa57e02b696ddaee342df0ee351", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.94.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4da6a74706fc7d93ab7252359727b9a6e8abadb0daf3145f8e35e4987182cd33", + "shape": [ + 32, + 16 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.94.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "fe326642f5d8ac0bf6744e78d1b2e26d5c0e0b171c72ed1fd298c03237d49b33", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.94.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "02342ec0914565cfc02eb2582ab4cec8124015b172da83156ffadbd3ae458492", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.experts.94.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d2f39e633c81c789e79b45128dec07fdadd2779df6fc343af33c81d070447bca", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.7.mlp.experts.94.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2572f891c4f386ff3013f6d797ab3476d3dcdb0dd9e4a653a19437849ec3ee2d", + "shape": [ + 16, + 32 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.7.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "59acf3a74198a92b69a142112a538d4654ad1b7904fb2cedc94234aea4448ab1", + "shape": [ + 288 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.7.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "4afa2aaa09e144b2900872521a76acad4f3ca698f1219a741ed069803b3d03a3", + "shape": [ + 288, + 4096 + ], + "shard": "model-00059-of-00062.safetensors", + "storage_bytes": 2359296 + }, + "model.language_model.layers.9.mlp.experts.129.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "e5cb0c58aa88b981e68807dc4a80db6ca030c26e9780aee5607b344ae6307b1e", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.129.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "662e03a2aa84d775496b58fa5e6c2c04073dd6db956e2df7238f6360eba76993", + "shape": [ + 32, + 16 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.129.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "46c9dd18b792d597b3f186a8e0e56d39898edf27178c8eb6bd912ba03bad835e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.129.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e636bfc141b11c788bc12be101648b8c16f5d48514dbb2b1235c4b383cf70831", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.129.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7c74d4b98d0d2c7ee273ed2e50c9992b0d5a628d55f9d232bea3fe1361507c64", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.129.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "0e9947d89a51a38626d8697b435f6f50b0d6cd18f53849f0b11ad34151944b9a", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.142.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "01efea53538459e01293e21050c16b0a275e83a80863b933d9f6ca0b9cf1ad6f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.142.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "aaac62ecd25cd24da3008265d2d21d1aee57a17e6e5615a0953aeeec45d1a19f", + "shape": [ + 32, + 16 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.142.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cfb1643eb893c124fb0481b5283bbcc45ca705c136bcc153875ff44187a9ac4e", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.142.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3e20a84afeee123414e586d80f720879eb5548aa6646b56ab194ad329c76d697", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.142.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "cd3c7d789bfc8c3782ece08fb1ce3ca236c65c031fbe542d1c6a55ee8b24191c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.142.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8692099e11bbc6dcde5ac3d27540c20ac761125ca9faddc7746b59ee36d6c5b6", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.150.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5284326529bb59f75e0696e050ccb4b2231bfc8110dc8625362b073d90b2eeae", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.150.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8b10e784285805a93f34b2179539ab62b8628f84dc2093894972ff10274b84b4", + "shape": [ + 32, + 16 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.150.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ee17017a41855287db3935f981a299851f35b655020ed420f7f2a2a652d66991", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.150.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "7da25cae2bd95799453e29444b44f36535deb4c60647c430eb800db97d10ba5d", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.150.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40efd19eaf4a43a3a69e1463fdeeef546430ec243707c44a5117750fc0822eb1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.150.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "32e84336ef7e62217e5355ce46cff29eddf4b79ba4e85ae1fd4a068a5c000124", + "shape": [ + 16, + 32 + ], + "shard": "model-00060-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.177.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9b2187dd1b90edd6c44920049678161a70339975d80059479f8618bb1d57af82", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.177.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14743931ce00a9791f6836bb8a99f7cf07fe247bea40b692eb6e097491132207", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.177.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ae2a0953297f3ca5956f650dbd919c7b2b2260ee5d395da1353caa9c0cdfc7f1", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.177.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9c20648c5dae8b9ec9142cde307c1e5d6858b15d7573ca0fe2ca86bc8e938ba2", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.177.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ea481b6704422d8245b5f55b7e926529f82851bf8b970f0758087bc103eeed99", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.177.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "687d42e128e3d10b0a9e35c570b06e64c007a57ff7d88b003b073fe42e479041", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.181.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "825ae168647f4fd136a81648af6b66cd4648364156444cb0eccbeb0d16633e87", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.181.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "94eaaf1be68eeeb54788a32f54fe2f0c368f7b8c8b8a0a21dd1675842b68eb3d", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.181.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "869dfa74f6ecc2f644187e3595b2a7c0af53024cf6e6de263d49db34847e5789", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.181.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9125862f4dae01da81a29e58a4d4ce0878d32ec75d7f1c15eadd3c820ceffe00", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.181.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "76a44684238625b9af10129bec2086b440962875a1cd59236b2254ee91f12d0f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.181.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "241fb3b285bd509bd46ec58f5aceff04841ad7e54684f659af4d94be2e2b5716", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.202.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "357575d2decc6c513447ef9c3b3ec8ac2bc94a7b8efc4ef8cfe12e2bbabda4f6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.202.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9ba4da2242ca5b0178675b26ab8eb320fe8b78b49848cbbf7c7808664c7c8cd5", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.202.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6cd2a5ad93483f19d76cf41a2a23cc75d7b369d36432878f7d2da21e68e7c31f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.202.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d80d7887668da5238ea7082a2a59e0c897d226dbf5e5e3d38291d591479692b1", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.202.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9b7a58eeee2e6d3407ca2b5886a5e7407bad44a3474eb8701c0270722c541f4b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.202.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "4fe19d598612f6b52a6abfd0f2edbd4712e1e2a797ed07bafc52ce776bf1c907", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.219.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "d031857018f19258f0c325334e6d41384dd24d5bce597a409d7a213458930115", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.219.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "116df644c2dc08ad0ea1b0d09225333824322c0f99e96881058377d43bec09fe", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.219.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6295f7ba873d5b10bbff825e2f21c907f27127f1df1c8cf82ad963f43f0c337a", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.219.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "124ff1559b1150f5c30f4008534fbd5badfa630ca1a23dee0d1bbcf7114c12bf", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.219.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "40d550ad6480d9ef5495d326103417cef978494f547970f51a25b4f7c941ac3c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.219.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "14e03e6400791c34802d13b5d743e236f52ba86aa7bb37924115e51d2688403b", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.221.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "52df001376edfaf01432ae80f57f96ec9e77385b5e5443bc002a88f269a3b6a6", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.221.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "98ec0e6738c8240ac1e88323717e1cae2fd1dbcaa239afeb5232bb03fd332884", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.221.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a1b6c62381b1cf0a184002ca09fcb76254fe691d39ac126008e87ad8679af0e0", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.221.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "6c27ef9faaebd96c34d7be80e25cf9faf0a2030be307a10f53ff3887544ebf4e", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.221.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3836df8871c9536eed22e34a7eca40a65fa11c674d48883004936766b1e370e4", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.221.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "86b29bc291a1f695c7e330375165876de374a0bcbae5bfc37ce26a6be1136d40", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.239.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "5ddbf0510bd5fde9677cc7cad8e963f02b77b38b2919357300e67ae44accbfcf", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.239.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "f04d2fc29b4d17cdbfe67e8307c14d04098f8b1a70951d709b1c720e5cfa6906", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.239.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "7e11fecb0c1a696a86354a20b6c843842603c6161b3336440bb3e108276fb43b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.239.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "8fd1b8dad2fe43471daee06764c31519f278207ee03b5d485feb3c97200380a8", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.239.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "9c503f0177f0f11d43982980f00174240dc0729d0419fd41d1aae4a92918b62c", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.239.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "96f8a0f4f4da70f917d6900287c3529befc188b019432df3a4a0fd41f15130c1", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.271.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6a91aaa90ce0b7a3609208c26f5928820175662838ef0fc1a0387fd2ecb5ef35", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.271.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "ad4943b48fc3286c8030b4daabd3746ecfb82521a384b2a0372127d6121849dd", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.271.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "1f61589c60397f09afab92e38bcf1c0760b3977e12938cb5a0a5ec7441ad8433", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.271.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3b9ad3dada06e1d671a581944ce644d9a6c4c71429b133241029d3041f772c20", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.271.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6cede3c1ecc8cb867df93dcce7d810952753443e7aa0c8a930ebff6c6e2cbf62", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.271.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "68b5cac7fa6b95abb56938c28a0fedcb8dbfa368047d91e607f2712cad091f5a", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.6.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "6bf0e50b57656ed050a3a5fdd6c7367218e9238ddf19f94037526e8758705cab", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.6.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "be01dd3e5eb5890c0b41131f5e3e7342c2877069c53e1a6c70cb1f4843efd423", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.6.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "b85a4c957b8f30869b16bd1f90b447e30c35a24673e3c70dd3109a97c53bc33b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.6.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2e10c7f75f243c857c0a046714f5e1c225c400ae5c24701fefb3ca70ba4e2206", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.6.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f64fd5efa553e395a1ca952bbaf88575642ac0ddedf7692d6dd756755dd10143", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.6.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c11209616983906689b42afd34088dd048794ea16685f9b8181c30bac2f8fa0a", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.74.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a37d5f5176d76db863fef3e4259534655f16a22fe0d4f0a2b78fabd5b46e9276", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.74.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "c33b655ece83f35ecda1274f7b59832e4e8ded651d1d2b5731cce3db5a2aadb8", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.74.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8ed77a08ef54d14a8c7ad9000a48f674f2f213723272a4299342224aad86a409", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.74.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d3b8b2e0a0696517295d7a3adb75ee51bda406fcc878893cdeb3d43ba31120ed", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.74.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "de8aec8238b4ade4932e875e04617311f10b38098c34a2f98dbd52d9f77092b9", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.74.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d19368d9e262b4d918307a9426010989884ae789ca038c300a4ce777a3df6da8", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.82.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "092b165bedaa3f679adb99f52a3f500df948b6d175d7a63eaa831a58ea5c1d78", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.82.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3399cc0836bee6c1765cb20f0b0b0ed3a38dfec4efb1654bda9d07a425038a2d", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.82.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ce376e334d7d0cb3665493d68b8f873f91ac1d837af22b1cdb149dcce94b5355", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.82.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "9211e4d87e3f947abd7a921c2f6b2e6789975167c7ba604ab17230447787e189", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.82.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "ab4d9db6ac0548197b475a6e1633a466c15d3f337dd5f90330f6683c063f3053", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.82.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "04839c4a521c8d3c1e0f255d302806b30fb8e6ab927f34acfb3bb99faa426a0b", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.88.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "12e6f4c3bd06c4c00c49048373c5c0bd9880958c34225a2676bafdf109798cc1", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.88.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "34ce6d8bb88d4a3425ef7a96fa7e1fc019bc3bc6d36df47fa4b92d554062ca18", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.88.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "66d2cf81a4b70ed1b4ca938edad037761375a1f2914f57ee269cab9d0742f26b", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.88.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "d4b9ffd2f7a59c07e1e0d57e30b446bb16be0aff645e07ede1c1e18fce6795da", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.88.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "4d807874bdd592a40758fd86ef3dc107f3d5a62b40199ada773566f55269dfaf", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.88.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "a4850c9d0c9c5fb552cddef4c42db4f5854984450b71f865c4551c151c7680a8", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.95.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "23a31723d51a4f94f082e892ebec77b7b03880bdcfc472d425e6466c83e2fa3f", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.95.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "2957f25fe35351a97c271b213deab1a3549aa6ebf74e3a0a4e11c4c7deaa85f1", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.95.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "33d52b1f17475dff810c053866b149b17f021363ad8b256e24509b4e57bbc6a2", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.95.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "dfc4e3ed598cb309ed4aa97f6a534fc37b9a2cfe6077ef9495903ffda24cc45f", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.95.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "a54a4f3601377580b40924bed75da3cfd24f701747f10e888ae39f07c342eb2f", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.95.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "74d8be410807ac228c038f7715455cbee6dafa8708a8c4b5786193efaf1d0473", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.98.down_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "8d6d63949970487f230385ff9a9a6c68390ef9b6ee1bc62fad73a878a06aaf86", + "shape": [ + 4096, + 2048 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.98.down_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "e56342e400985d8b3e67be0e91acbe28a54227ee8be4b6db7e4c166e9bccb08f", + "shape": [ + 32, + 16 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.98.gate_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "f8094f2b00522762ea63d80b98c51db2cef5b988cb219fa3d2c47a13ddfa2b07", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.98.gate_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "1b516a03733892ae6d68a622f9f77ca78cdddaa6c8f601c8e17afa1161b3ce3d", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.experts.98.up_proj.weight": { + "dtype": "torch.float8_e4m3fn", + "numel": 8388608, + "sha256": "3ddc79ae6c2485649fc0d90499af4f619682a5b9aaff55beb4641d81716dd206", + "shape": [ + 2048, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 8388608 + }, + "model.language_model.layers.9.mlp.experts.98.up_proj.weight_scale_inv": { + "dtype": "torch.float32", + "numel": 512, + "sha256": "3a4a61fc7c43ff06b528a980d2c99f22140107d495170884c1571809501dbadc", + "shape": [ + 16, + 32 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2048 + }, + "model.language_model.layers.9.mlp.gate.e_score_correction_bias": { + "dtype": "torch.float32", + "numel": 288, + "sha256": "19f20ec9f717e908de5842170c86714af13a7efb6592e25e69e7fe43aff00780", + "shape": [ + 288 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 1152 + }, + "model.language_model.layers.9.mlp.gate.weight": { + "dtype": "torch.bfloat16", + "numel": 1179648, + "sha256": "d4607afaaec335d6913df0d344bae4b0d4bc7af3c6fca188b068fd363b482acb", + "shape": [ + 288, + 4096 + ], + "shard": "model-00061-of-00062.safetensors", + "storage_bytes": 2359296 + } + } + } +}