manaestras commited on
Commit
b7bf724
·
verified ·
1 Parent(s): 449e7d4

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +18 -131
config.json CHANGED
@@ -2,188 +2,75 @@
2
  "architectures": [
3
  "HunYuanVLForConditionalGeneration"
4
  ],
5
- "attention_bias": false,
6
- "attention_dropout": 0.0,
7
- "attention_head_dim": 128,
8
- "bos_token_id": 120000,
9
  "dtype": "bfloat16",
10
- "eod_token_id": 120020,
11
- "eos_token_id": 120020,
12
- "head_dim": 128,
13
- "hidden_act": "silu",
14
- "hidden_size": 1024,
15
- "image_end_token_id": 120119,
16
- "image_newline_token_id": 120121,
17
- "image_start_token_id": 120118,
18
  "image_token_id": 120120,
19
- "initializer_range": 0.02,
20
- "intermediate_size": 3584,
21
- "max_position_embeddings": 131072,
22
- "mlp_bias": false,
23
  "model_type": "hunyuan_vl",
24
- "norm_type": "rms",
25
- "num_attention_heads": 16,
26
- "num_experts": 1,
27
- "num_hidden_layers": 24,
28
- "num_key_value_heads": 8,
29
- "org_vocab_size": 120818,
30
- "pad_id": 120002,
31
- "pad_token_id": 120002,
32
- "pretraining_tp": 1,
33
- "rms_norm_eps": 1e-05,
34
- "rope_scaling": {
35
- "alpha": 1000.0,
36
- "beta_fast": 32,
37
- "beta_slow": 1,
38
- "factor": 1.0,
39
- "mscale": 1.0,
40
- "mscale_all_dim": 1.0,
41
- "type": "xdrope",
42
- "xdrope_section": [
43
- 16,
44
- 16,
45
- 16,
46
- 16
47
- ]
48
- },
49
- "rope_theta": 10000.0,
50
- "routed_scaling_factor": 1.0,
51
- "sep_token_id": 0,
52
  "text_config": {
53
- "add_classification_head": false,
54
- "architectures": [
55
- "HunYuanVLForConditionalGeneration"
56
- ],
57
  "attention_bias": false,
58
  "attention_dropout": 0.0,
59
- "attention_head_dim": 128,
60
  "bos_token_id": 120000,
61
- "class_num": 0,
62
- "dense_list": [
63
- 1024,
64
- 0
65
- ],
66
- "dtype": "bfloat16",
67
  "eod_token_id": 120020,
68
- "eos_token_id": 120007,
69
- "expert_hidden_dim": null,
70
- "first_k_dense_replace": 0,
71
- "group_limited_greedy": false,
72
  "head_dim": 128,
73
  "hidden_act": "silu",
74
  "hidden_size": 1024,
75
- "image_newline_token_id": 120121,
76
  "initializer_range": 0.02,
77
  "intermediate_size": 3584,
78
- "kv_lora_rank": 512,
79
- "mask_init_id": 13,
80
  "max_position_embeddings": 131072,
81
- "mlp_bias": false,
82
  "model_type": "hunyuan_vl_text",
83
- "moe_drop_tokens": false,
84
- "moe_intermediate_size": null,
85
- "moe_layer_num_skipped": 0,
86
- "moe_random_routing_dropped_token": false,
87
- "moe_topk": 1,
88
- "mtp_loss_factor": 0.1,
89
- "mtp_no_bias": true,
90
- "n_group": null,
91
- "norm_topk_prob": true,
92
- "norm_type": "rms",
93
  "num_attention_heads": 16,
94
- "num_experts": 1,
95
- "num_experts_per_tok": 1,
96
  "num_hidden_layers": 24,
97
  "num_key_value_heads": 8,
98
- "num_nextn_predict_layers": 1,
99
- "num_predictor_layers": 0,
100
- "num_shared_expert": 1,
101
- "num_shared_experts": 1,
102
- "org_vocab_size": 120818,
103
- "pad_id": 120002,
104
  "pad_token_id": 120002,
105
- "pool_type": "last",
106
  "pretraining_tp": 1,
107
- "q_lora_rank": 1536,
108
- "qk_nope_head_dim": 128,
109
- "qk_norm": false,
110
- "qk_rope_head_dim": 64,
111
  "rms_norm_eps": 1e-05,
112
- "rope_scaling": {
113
  "alpha": 1000.0,
114
  "beta_fast": 32,
115
  "beta_slow": 1,
116
  "factor": 1.0,
117
- "mscale": 1.0,
118
- "mscale_all_dim": 1.0,
119
- "type": "xdrope",
120
- "xdrope_section": [
121
  16,
122
  16,
123
  16,
124
  16
125
- ]
 
 
 
 
 
126
  },
127
- "rope_theta": 10000.0,
128
- "routed_scaling_factor": 1.0,
129
  "sep_token_id": 0,
130
- "text_end_id": 8,
131
- "text_start_id": 7,
132
  "tie_word_embeddings": true,
133
- "topk_group": null,
134
  "use_cache": true,
135
- "use_cla": false,
136
- "use_mixed_mlp_moe": false,
137
- "use_mla": false,
138
- "use_qk_norm": true,
139
- "use_rotary_pos_emb": true,
140
- "v_head_dim": 128,
141
- "vision_full_attention": false,
142
  "vocab_size": 120818
143
  },
144
- "text_end_id": 8,
145
- "text_start_id": 7,
146
- "transformers_version": "4.57.1",
147
- "use_cache": true,
148
- "use_cla": false,
149
- "use_qk_norm": true,
150
- "video_end_token_id": 120123,
151
- "video_start_token_id": 120122,
152
  "vision_config": {
153
- "add_patchemb_bias": true,
154
- "anyres_vit_max_image_size": 2048,
155
  "attention_dropout": 0.0,
156
- "cat_extra_token": 1,
157
- "dtype": "bfloat16",
158
  "hidden_act": "gelu",
159
- "hidden_dropout": 0.0,
160
  "hidden_size": 1152,
161
  "img_max_token_num": 16384,
162
  "intermediate_size": 4304,
163
  "interpolate_mode": "bilinear",
164
- "learnable_mlp_pooling_size": 0,
165
  "max_image_size": 2048,
166
  "max_vit_seq_len": 65536,
167
  "min_image_size": 512,
168
- "model_type": "hunyuan_vl",
169
  "num_attention_heads": 16,
170
  "num_channels": 3,
171
  "num_hidden_layers": 27,
172
  "num_key_value_heads": 16,
173
  "out_hidden_size": 1024,
174
  "patch_size": 16,
175
- "perceive_post_norm": true,
176
- "perceive_pre_norm": true,
177
- "remove_prenorm": true,
178
- "resize_resolution": 2048,
179
  "rms_norm_eps": 1e-05,
180
  "spatial_merge_size": 2,
181
- "spatial_patch_size": 1,
182
  "temporal_patch_size": 1,
183
- "text_hidden_size": 1024,
184
- "video_max_image_size": 768,
185
- "video_min_image_size": 256,
186
- "vision_full_attention": false
187
- },
188
- "vocab_size": 120818
189
- }
 
2
  "architectures": [
3
  "HunYuanVLForConditionalGeneration"
4
  ],
 
 
 
 
5
  "dtype": "bfloat16",
6
+ "im_end_id": 120119,
7
+ "im_newline_id": 120121,
8
+ "im_start_id": 120118,
 
 
 
 
 
9
  "image_token_id": 120120,
 
 
 
 
10
  "model_type": "hunyuan_vl",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  "text_config": {
 
 
 
 
12
  "attention_bias": false,
13
  "attention_dropout": 0.0,
 
14
  "bos_token_id": 120000,
 
 
 
 
 
 
15
  "eod_token_id": 120020,
16
+ "eos_token_id": 120020,
 
 
 
17
  "head_dim": 128,
18
  "hidden_act": "silu",
19
  "hidden_size": 1024,
 
20
  "initializer_range": 0.02,
21
  "intermediate_size": 3584,
 
 
22
  "max_position_embeddings": 131072,
 
23
  "model_type": "hunyuan_vl_text",
 
 
 
 
 
 
 
 
 
 
24
  "num_attention_heads": 16,
 
 
25
  "num_hidden_layers": 24,
26
  "num_key_value_heads": 8,
 
 
 
 
 
 
27
  "pad_token_id": 120002,
 
28
  "pretraining_tp": 1,
 
 
 
 
29
  "rms_norm_eps": 1e-05,
30
+ "rope_parameters": {
31
  "alpha": 1000.0,
32
  "beta_fast": 32,
33
  "beta_slow": 1,
34
  "factor": 1.0,
35
+ "mrope_section": [
 
 
 
36
  16,
37
  16,
38
  16,
39
  16
40
+ ],
41
+ "mscale": 1.0,
42
+ "mscale_all_dim": 1.0,
43
+ "rope_theta": 10000.0,
44
+ "rope_type": "dynamic",
45
+ "type": "dynamic"
46
  },
 
 
47
  "sep_token_id": 0,
 
 
48
  "tie_word_embeddings": true,
 
49
  "use_cache": true,
 
 
 
 
 
 
 
50
  "vocab_size": 120818
51
  },
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.15.0.dev0",
 
 
 
 
 
 
54
  "vision_config": {
 
 
55
  "attention_dropout": 0.0,
 
 
56
  "hidden_act": "gelu",
 
57
  "hidden_size": 1152,
58
  "img_max_token_num": 16384,
59
  "intermediate_size": 4304,
60
  "interpolate_mode": "bilinear",
 
61
  "max_image_size": 2048,
62
  "max_vit_seq_len": 65536,
63
  "min_image_size": 512,
64
+ "model_type": "hunyuan_vl_vision",
65
  "num_attention_heads": 16,
66
  "num_channels": 3,
67
  "num_hidden_layers": 27,
68
  "num_key_value_heads": 16,
69
  "out_hidden_size": 1024,
70
  "patch_size": 16,
 
 
 
 
71
  "rms_norm_eps": 1e-05,
72
  "spatial_merge_size": 2,
 
73
  "temporal_patch_size": 1,
74
+ "text_hidden_size": 1024
75
+ }
76
+ }