Upload folder using huggingface_hub

README.md

@@ -6,28 +6,34 @@ tags:
 - lazymergekit
 - automerger
 base_model:
+- rwitz/experiment26-truthy-iter-1
 - Gille/StrangeMerges_32-7B-slerp
 ---
 
 # Experiment26Strangemerges_32-7B
 
 Experiment26Strangemerges_32-7B is an automated merge created by [Maxime Labonne](https://huggingface.co/mlabonne) using the following configuration.
+* [rwitz/experiment26-truthy-iter-1](https://huggingface.co/rwitz/experiment26-truthy-iter-1)
 * [Gille/StrangeMerges_32-7B-slerp](https://huggingface.co/Gille/StrangeMerges_32-7B-slerp)
 
 ## 🧩 Configuration
 
 ```yaml
-models:
-- model: rwitz/experiment26-truthy-iter-1
-  # No parameters necessary for base model
-- model: Gille/StrangeMerges_32-7B-slerp
-  parameters:
-    density: 0.53
-    weight: 0.6
-merge_method: dare_ties
+slices:
+  - sources:
+      - model: rwitz/experiment26-truthy-iter-1
+        layer_range: [0, 32]
+      - model: Gille/StrangeMerges_32-7B-slerp
+        layer_range: [0, 32]
+merge_method: slerp
 base_model: rwitz/experiment26-truthy-iter-1
 parameters:
-int8_mask: true
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
 dtype: bfloat16
 random_seed: 0
     ```

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.39.1",
   "use_cache": true,
   "vocab_size": 32000
 }

mergekit_config.yml

@@ -1,15 +1,19 @@
 
-models:
-- model: rwitz/experiment26-truthy-iter-1
-  # No parameters necessary for base model
-- model: Gille/StrangeMerges_32-7B-slerp
-  parameters:
-    density: 0.53
-    weight: 0.6
-merge_method: dare_ties
+slices:
+  - sources:
+      - model: rwitz/experiment26-truthy-iter-1
+        layer_range: [0, 32]
+      - model: Gille/StrangeMerges_32-7B-slerp
+        layer_range: [0, 32]
+merge_method: slerp
 base_model: rwitz/experiment26-truthy-iter-1
 parameters:
-int8_mask: true
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
 dtype: bfloat16
 random_seed: 0
     

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb65069e8b9708b103b099e982bdfa3bf42783de43f3912c0c7b0403e4ae9e4d
+oid sha256:43f832134dcee360302da1ec4ace3f8f297992874a30ab895bc7cb6f3fb90fef
 size 9942981696

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15e5e3348e6e2b8fcea6273256d17f3e249417a764d7fd66029649731d85a0a9
+oid sha256:bb187664697ed91571d4b2b812b98de7245ed9964697729ca0d0229f7c67a73f
 size 4540516344