akkikiki commited on
Commit
eb07696
·
verified ·
1 Parent(s): d426143

Fix NVFP4 quantization metadata: stale per-group mxfp4 format, glob-style exclude_modules

Browse files

The generated config still described the MXFP4 source in three places, each of which makes the checkpoint unloadable:

1. `config_groups.group_0.format` was still `mxfp4-pack-quantized`. SGLang's Kimi-K3 argument override keys on this and forces `moe_runner_backend=flashinfer_mxfp4`, i.e. MXFP4 kernels over NVFP4 tensors.
2. `ignore` used compressed-tensors `re:` regexes. ModelOpt matches with `re.fullmatch(pattern.replace('.', r'\.').replace('*', '.*'))`, so those patterns never match; and it has no `exclude_modules` key at all. Since ModelOpt quantizes everything *except* `exclude_modules` (the inverse of compressed-tensors), unquantized modules were loaded as NVFP4 and failed a shape assertion.
3. `weights` still carried MXFP4 block/scale metadata.

`exclude_modules` is now derived from the modules that actually kept an unpacked 2-D float `.weight` in the shards (2,398 of them, collapsed to 55 globs), verified to cover all of them while excluding none of the 247,296 quantized modules.

Files changed (2) hide show
  1. config.json +114 -8
  2. hf_quant_config.json +57 -7
config.json CHANGED
@@ -202,7 +202,7 @@
202
  "quantization_config": {
203
  "config_groups": {
204
  "group_0": {
205
- "format": "mxfp4-pack-quantized",
206
  "input_activations": null,
207
  "output_activations": null,
208
  "targets": [
@@ -227,19 +227,125 @@
227
  "format": "nvfp4-pack-quantized",
228
  "global_compression_ratio": null,
229
  "ignore": [
230
- "re:.*self_attn.*",
231
- "re:.*shared_experts.*",
232
- "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",
233
- "re:.*lm_head.*",
234
- "re:.*vision_tower.*",
235
- "re:.*mm_projector.*"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
236
  ],
237
  "kv_cache_scheme": null,
238
  "quant_method": "modelopt",
239
  "quantization_status": "compressed",
240
  "group_size": 16,
241
  "quant_algo": "NVFP4",
242
- "kv_cache_quant_algo": null
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
243
  },
244
  "remove_invalid_values": false,
245
  "repetition_penalty": 1.0,
 
202
  "quantization_config": {
203
  "config_groups": {
204
  "group_0": {
205
+ "format": "nvfp4-pack-quantized",
206
  "input_activations": null,
207
  "output_activations": null,
208
  "targets": [
 
227
  "format": "nvfp4-pack-quantized",
228
  "global_compression_ratio": null,
229
  "ignore": [
230
+ "*self_attn*",
231
+ "*shared_experts*",
232
+ "*mlp.gate_proj*",
233
+ "*mlp.up_proj*",
234
+ "*mlp.gate_up_proj*",
235
+ "*mlp.down_proj*",
236
+ "*lm_head*",
237
+ "*vision_tower*",
238
+ "*mm_projector*",
239
+ "*language_model.lm_head*",
240
+ "*language_model.model.embed_tokens*",
241
+ "*input_layernorm*",
242
+ "*mlp_res_norm*",
243
+ "*mlp_res_proj*",
244
+ "*post_attention_layernorm*",
245
+ "*self_attention_res_norm*",
246
+ "*self_attention_res_proj*",
247
+ "*self_attn.b_proj*",
248
+ "*self_attn.f_a_proj*",
249
+ "*self_attn.f_b_proj*",
250
+ "*self_attn.g_proj*",
251
+ "*self_attn.k_conv1d*",
252
+ "*self_attn.k_proj*",
253
+ "*self_attn.o_norm*",
254
+ "*self_attn.o_proj*",
255
+ "*self_attn.q_conv1d*",
256
+ "*self_attn.q_proj*",
257
+ "*self_attn.v_conv1d*",
258
+ "*self_attn.v_proj*",
259
+ "*block_sparse_moe.gate*",
260
+ "*block_sparse_moe.routed_expert_down_proj*",
261
+ "*block_sparse_moe.routed_expert_norm*",
262
+ "*block_sparse_moe.routed_expert_up_proj*",
263
+ "*block_sparse_moe.shared_experts.down_proj*",
264
+ "*block_sparse_moe.shared_experts.gate_proj*",
265
+ "*block_sparse_moe.shared_experts.up_proj*",
266
+ "*self_attn.kv_a_layernorm*",
267
+ "*self_attn.kv_a_proj_with_mqa*",
268
+ "*self_attn.kv_b_proj*",
269
+ "*self_attn.q_a_layernorm*",
270
+ "*self_attn.q_a_proj*",
271
+ "*self_attn.q_b_proj*",
272
+ "*language_model.model.norm*",
273
+ "*language_model.model.output_attn_res_norm*",
274
+ "*language_model.model.output_attn_res_proj*",
275
+ "*mm_projector.post_norm*",
276
+ "*mlp.fc0*",
277
+ "*mlp.fc1*",
278
+ "*norm0*",
279
+ "*norm1*",
280
+ "*wo*",
281
+ "*wqkv*",
282
+ "*vision_tower.encoder.final_layernorm*",
283
+ "*vision_tower.patch_embed.pos_emb*",
284
+ "*vision_tower.patch_embed.proj*"
285
  ],
286
  "kv_cache_scheme": null,
287
  "quant_method": "modelopt",
288
  "quantization_status": "compressed",
289
  "group_size": 16,
290
  "quant_algo": "NVFP4",
291
+ "kv_cache_quant_algo": null,
292
+ "exclude_modules": [
293
+ "*self_attn*",
294
+ "*shared_experts*",
295
+ "*mlp.gate_proj*",
296
+ "*mlp.up_proj*",
297
+ "*mlp.gate_up_proj*",
298
+ "*mlp.down_proj*",
299
+ "*lm_head*",
300
+ "*vision_tower*",
301
+ "*mm_projector*",
302
+ "*language_model.lm_head*",
303
+ "*language_model.model.embed_tokens*",
304
+ "*input_layernorm*",
305
+ "*mlp_res_norm*",
306
+ "*mlp_res_proj*",
307
+ "*post_attention_layernorm*",
308
+ "*self_attention_res_norm*",
309
+ "*self_attention_res_proj*",
310
+ "*self_attn.b_proj*",
311
+ "*self_attn.f_a_proj*",
312
+ "*self_attn.f_b_proj*",
313
+ "*self_attn.g_proj*",
314
+ "*self_attn.k_conv1d*",
315
+ "*self_attn.k_proj*",
316
+ "*self_attn.o_norm*",
317
+ "*self_attn.o_proj*",
318
+ "*self_attn.q_conv1d*",
319
+ "*self_attn.q_proj*",
320
+ "*self_attn.v_conv1d*",
321
+ "*self_attn.v_proj*",
322
+ "*block_sparse_moe.gate*",
323
+ "*block_sparse_moe.routed_expert_down_proj*",
324
+ "*block_sparse_moe.routed_expert_norm*",
325
+ "*block_sparse_moe.routed_expert_up_proj*",
326
+ "*block_sparse_moe.shared_experts.down_proj*",
327
+ "*block_sparse_moe.shared_experts.gate_proj*",
328
+ "*block_sparse_moe.shared_experts.up_proj*",
329
+ "*self_attn.kv_a_layernorm*",
330
+ "*self_attn.kv_a_proj_with_mqa*",
331
+ "*self_attn.kv_b_proj*",
332
+ "*self_attn.q_a_layernorm*",
333
+ "*self_attn.q_a_proj*",
334
+ "*self_attn.q_b_proj*",
335
+ "*language_model.model.norm*",
336
+ "*language_model.model.output_attn_res_norm*",
337
+ "*language_model.model.output_attn_res_proj*",
338
+ "*mm_projector.post_norm*",
339
+ "*mlp.fc0*",
340
+ "*mlp.fc1*",
341
+ "*norm0*",
342
+ "*norm1*",
343
+ "*wo*",
344
+ "*wqkv*",
345
+ "*vision_tower.encoder.final_layernorm*",
346
+ "*vision_tower.patch_embed.pos_emb*",
347
+ "*vision_tower.patch_embed.proj*"
348
+ ]
349
  },
350
  "remove_invalid_values": false,
351
  "repetition_penalty": 1.0,
hf_quant_config.json CHANGED
@@ -8,13 +8,63 @@
8
  "kv_cache_quant_algo": null,
9
  "group_size": 16,
10
  "exclude_modules": [
11
- "re:.*lm_head.*",
12
- "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",
13
- "re:.*mm_projector.*",
14
- "re:.*self_attn.*",
15
- "re:.*shared_experts.*",
16
- "re:.*vision_tower.*"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  ],
18
- "quantized_modules_count": 247296
 
19
  }
20
  }
 
8
  "kv_cache_quant_algo": null,
9
  "group_size": 16,
10
  "exclude_modules": [
11
+ "*self_attn*",
12
+ "*shared_experts*",
13
+ "*mlp.gate_proj*",
14
+ "*mlp.up_proj*",
15
+ "*mlp.gate_up_proj*",
16
+ "*mlp.down_proj*",
17
+ "*lm_head*",
18
+ "*vision_tower*",
19
+ "*mm_projector*",
20
+ "*language_model.lm_head*",
21
+ "*language_model.model.embed_tokens*",
22
+ "*input_layernorm*",
23
+ "*mlp_res_norm*",
24
+ "*mlp_res_proj*",
25
+ "*post_attention_layernorm*",
26
+ "*self_attention_res_norm*",
27
+ "*self_attention_res_proj*",
28
+ "*self_attn.b_proj*",
29
+ "*self_attn.f_a_proj*",
30
+ "*self_attn.f_b_proj*",
31
+ "*self_attn.g_proj*",
32
+ "*self_attn.k_conv1d*",
33
+ "*self_attn.k_proj*",
34
+ "*self_attn.o_norm*",
35
+ "*self_attn.o_proj*",
36
+ "*self_attn.q_conv1d*",
37
+ "*self_attn.q_proj*",
38
+ "*self_attn.v_conv1d*",
39
+ "*self_attn.v_proj*",
40
+ "*block_sparse_moe.gate*",
41
+ "*block_sparse_moe.routed_expert_down_proj*",
42
+ "*block_sparse_moe.routed_expert_norm*",
43
+ "*block_sparse_moe.routed_expert_up_proj*",
44
+ "*block_sparse_moe.shared_experts.down_proj*",
45
+ "*block_sparse_moe.shared_experts.gate_proj*",
46
+ "*block_sparse_moe.shared_experts.up_proj*",
47
+ "*self_attn.kv_a_layernorm*",
48
+ "*self_attn.kv_a_proj_with_mqa*",
49
+ "*self_attn.kv_b_proj*",
50
+ "*self_attn.q_a_layernorm*",
51
+ "*self_attn.q_a_proj*",
52
+ "*self_attn.q_b_proj*",
53
+ "*language_model.model.norm*",
54
+ "*language_model.model.output_attn_res_norm*",
55
+ "*language_model.model.output_attn_res_proj*",
56
+ "*mm_projector.post_norm*",
57
+ "*mlp.fc0*",
58
+ "*mlp.fc1*",
59
+ "*norm0*",
60
+ "*norm1*",
61
+ "*wo*",
62
+ "*wqkv*",
63
+ "*vision_tower.encoder.final_layernorm*",
64
+ "*vision_tower.patch_embed.pos_emb*",
65
+ "*vision_tower.patch_embed.proj*"
66
  ],
67
+ "quantized_modules_count": 247296,
68
+ "activation_scheme": "none"
69
  }
70
  }