Hugging Face's logo

Hiibb
/
factor4

Model card Files
xet
Community
Hiibb commited on
Commit
80ce82c
·
verified ·
1 Parent(s): 4e41157

Upload folder using huggingface_hub

Browse files
Files changed (18) hide show
  1. resid_post_layer_12/trainer_0/ae.pt +3 -0
  2. resid_post_layer_12/trainer_0/config.json +53 -0
  3. resid_post_layer_12/trainer_0/eval_results.json +1 -0
  4. resid_post_layer_12/trainer_1/ae.pt +3 -0
  5. resid_post_layer_12/trainer_1/config.json +53 -0
  6. resid_post_layer_12/trainer_1/eval_results.json +1 -0
  7. resid_post_layer_12/trainer_2/ae.pt +3 -0
  8. resid_post_layer_12/trainer_2/config.json +53 -0
  9. resid_post_layer_12/trainer_2/eval_results.json +1 -0
  10. resid_post_layer_12/trainer_3/ae.pt +3 -0
  11. resid_post_layer_12/trainer_3/config.json +53 -0
  12. resid_post_layer_12/trainer_3/eval_results.json +1 -0
  13. resid_post_layer_12/trainer_4/ae.pt +3 -0
  14. resid_post_layer_12/trainer_4/config.json +53 -0
  15. resid_post_layer_12/trainer_4/eval_results.json +1 -0
  16. resid_post_layer_12/trainer_5/ae.pt +3 -0
  17. resid_post_layer_12/trainer_5/config.json +53 -0
  18. resid_post_layer_12/trainer_5/eval_results.json +1 -0
resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84ce780006ea7028a5ba8f88bfff9cff422a3068c06f3596468c68d9998cb849
3
+ size 302066957
resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 20,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 69.16, "l1_loss": 353.58, "l0": 19.8575830078125, "frac_variance_explained": 0.76376953125, "cossim": 0.87365234375, "l2_ratio": 0.8874609375, "relative_reconstruction_bias": 1.01203125, "loss_original": 2.152919921875, "loss_reconstructed": 2.48701171875, "loss_zero": 12.4375, "frac_recovered": 0.96708984375, "frac_alive": 0.88458251953125, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_1/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:430d1cb2ad5bac2526b3bad1926ac89a93f57f650f17d0812fec1b75160e904c
3
+ size 302066957
resid_post_layer_12/trainer_1/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 40,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_1",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_1/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 63.835, "l1_loss": 537.06, "l0": 39.9339794921875, "frac_variance_explained": 0.79654296875, "cossim": 0.8934375, "l2_ratio": 0.905859375, "relative_reconstruction_bias": 1.01015625, "loss_original": 2.152919921875, "loss_reconstructed": 2.3361328125, "loss_zero": 12.4375, "frac_recovered": 0.98150390625, "frac_alive": 0.92437744140625, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_2/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:426117f489e9785ddd299ccb3c8ee5b54f5cea8039b85c9be9bd765a889c98c4
3
+ size 302066957
resid_post_layer_12/trainer_2/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 80,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_2",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_2/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 58.7225, "l1_loss": 801.26, "l0": 79.705771484375, "frac_variance_explained": 0.8289453125, "cossim": 0.91025390625, "l2_ratio": 0.9239453125, "relative_reconstruction_bias": 1.0105078125, "loss_original": 2.152919921875, "loss_reconstructed": 2.257880859375, "loss_zero": 12.4375, "frac_recovered": 0.9891015625, "frac_alive": 0.9534912109375, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_3/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9923d4a9b23f156b4cc320127e84c61b9866c98d44efc9ce2c1c0b4f0c605f9a
3
+ size 302066957
resid_post_layer_12/trainer_3/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 160,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_3",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_3/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 53.61875, "l1_loss": 1259.32, "l0": 159.901748046875, "frac_variance_explained": 0.8572265625, "cossim": 0.92578125, "l2_ratio": 0.93794921875, "relative_reconstruction_bias": 1.0096875, "loss_original": 2.152919921875, "loss_reconstructed": 2.215146484375, "loss_zero": 12.4375, "frac_recovered": 0.99341796875, "frac_alive": 0.9521484375, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_4/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0bceee2473fbb3582343241ca9e51b2bdfaeb84ff043e36d523d4015572d6da
3
+ size 302066957
resid_post_layer_12/trainer_4/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 320,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_4",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_4/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 48.275, "l1_loss": 1988.88, "l0": 321.033984375, "frac_variance_explained": 0.88748046875, "cossim": 0.94056640625, "l2_ratio": 0.953046875, "relative_reconstruction_bias": 1.0080859375, "loss_original": 2.152919921875, "loss_reconstructed": 2.19083984375, "loss_zero": 12.4375, "frac_recovered": 0.99576171875, "frac_alive": 0.9088134765625, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_5/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0dc57770b3c3a04cbc8db3602ad69eb2fd223343a601848e4640acd060d50d5b
3
+ size 302066957
resid_post_layer_12/trainer_5/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.00390625,
18
+ 0.015625,
19
+ 0.0625,
20
+ 0.25,
21
+ 0.66796875
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 64,
32
+ 256,
33
+ 1024,
34
+ 4096,
35
+ 10944
36
+ ],
37
+ "k": 640,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_5",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_5/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 41.31375, "l1_loss": 3365.2, "l0": 642.4017333984375, "frac_variance_explained": 0.91783203125, "cossim": 0.95521484375, "l2_ratio": 0.964375, "relative_reconstruction_bias": 1.007421875, "loss_original": 2.152919921875, "loss_reconstructed": 2.173193359375, "loss_zero": 12.4375, "frac_recovered": 0.99751953125, "frac_alive": 0.721435546875, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}