namespace-Pt
commited on
Commit
•
b6abf69
1
Parent(s):
c06f207
Upload config.json with huggingface_hub
Browse files- config.json +2 -2
config.json
CHANGED
@@ -11,9 +11,9 @@
|
|
11 |
"attention_dropout": 0.0,
|
12 |
"beacon_attn": "step-expansion",
|
13 |
"beacon_layers": null,
|
|
|
14 |
"beacon_ratio": [2,4,8,16,32,64,128],
|
15 |
"beacon_ratio_mix": "adapt-1024",
|
16 |
-
"beacon_seed": 42,
|
17 |
"beacon_stride": [1024],
|
18 |
"beacon_stride_mix": "step-random",
|
19 |
"beacon_window": 1024,
|
@@ -23,7 +23,7 @@
|
|
23 |
"hidden_size": 4096,
|
24 |
"initializer_range": 0.02,
|
25 |
"intermediate_size": 11008,
|
26 |
-
"max_position_embeddings":
|
27 |
"model_type": "llama",
|
28 |
"num_attention_heads": 32,
|
29 |
"num_hidden_layers": 32,
|
|
|
11 |
"attention_dropout": 0.0,
|
12 |
"beacon_attn": "step-expansion",
|
13 |
"beacon_layers": null,
|
14 |
+
"beacon_param": ["q", "k", "v", "o"],
|
15 |
"beacon_ratio": [2,4,8,16,32,64,128],
|
16 |
"beacon_ratio_mix": "adapt-1024",
|
|
|
17 |
"beacon_stride": [1024],
|
18 |
"beacon_stride_mix": "step-random",
|
19 |
"beacon_window": 1024,
|
|
|
23 |
"hidden_size": 4096,
|
24 |
"initializer_range": 0.02,
|
25 |
"intermediate_size": 11008,
|
26 |
+
"max_position_embeddings": 4096,
|
27 |
"model_type": "llama",
|
28 |
"num_attention_heads": 32,
|
29 |
"num_hidden_layers": 32,
|