Update README.md

README.md

@@ -105,4 +105,708 @@ I welcome contributions from the you! To contribute to J.O.S.I.E., please fork t
 
 ## License
 
-J.O.S.I.E. is licensed under the Apache2 License. See the [LICENSE](LICENSE) file for more details.
+J.O.S.I.E. is licensed under the Apache2 License. See the [LICENSE](LICENSE) file for more details.
+
+
+
+
+
+# Big Updates!
+
+I have finaly trained the Vision and Audio encoder part, big thangs to FaceBook Research for the ImageBind model, wich is what I have build it on top of.
+
+What I did was, I copied the weights from the original ImageBind model into a second 'downcycled' ImageBindVisionAudioHuge model.
+After that I have continued to trained the model on a custom Vision and Audio dataset using the contrastive learning Algorythm introduced by Google with Pali Gemma with the text embeddings from the origional ImageBind model.
+
+After mergind the encoder with the test reasoner (Qwen2-0.5B-Instruct), I got succesfull inference on both video, image and audio.
+I will slowly start writing the training scrypt, creating the new dataset, and optimizing the model and inference code a litle bit more, and lastly train the model.
+
+Here are the actual model layers:
+
+```txt
+ImageBindModelAudioVision(
+  (modality_preprocessors): ModuleDict(
+    (vision): RGBDTPreprocessor(
+      (cls_token): tensor((1, 1, 1280), requires_grad=True)
+      
+      (rgbt_stem): PatchEmbedGeneric(
+        (proj): Sequential(
+          (0): PadIm2Video()
+          (1): Conv3d(3, 1280, kernel_size=(2, 14, 14), stride=(2, 14, 14), bias=False)
+        )
+      )
+      (pos_embedding_helper): SpatioTemporalPosEmbeddingHelper(
+        (pos_embed): tensor((1, 257, 1280), requires_grad=True)
+        
+      )
+    )
+    (audio): AudioPreprocessor(
+      (cls_token): tensor((1, 1, 768), requires_grad=True)
+      
+      (rgbt_stem): PatchEmbedGeneric(
+        (proj): Conv2d(1, 768, kernel_size=(16, 16), stride=(10, 10), bias=False)
+        (norm_layer): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+      )
+      (pos_embedding_helper): SpatioTemporalPosEmbeddingHelper(
+        (pos_embed): tensor((1, 229, 768), requires_grad=True)
+        
+      )
+    )
+  )
+  (modality_trunks): ModuleDict(
+    (vision): SimpleTransformer(
+      (pre_transformer_layer): Sequential(
+        (0): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        (1): EinOpsRearrange()
+      )
+      (blocks): Sequential(
+        (0): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (1): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (2): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (3): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (4): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (5): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (6): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (7): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (8): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (9): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (10): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (11): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (12): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (13): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (14): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (15): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (16): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (17): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (18): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (19): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (20): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (21): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (22): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (23): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (24): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (25): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (26): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (27): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (28): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (29): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (30): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+        (31): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=1280, out_features=1280, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1280, out_features=5120, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=5120, out_features=1280, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+        )
+      )
+      (post_transformer_layer): EinOpsRearrange()
+    )
+    (audio): SimpleTransformer(
+      (pre_transformer_layer): Sequential(
+        (0): Identity()
+        (1): EinOpsRearrange()
+      )
+      (blocks): Sequential(
+        (0): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): Identity()
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (1): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.009)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (2): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.018)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (3): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.027)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (4): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.036)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (5): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.045)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (6): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.055)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (7): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.064)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (8): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.073)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (9): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.082)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (10): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.091)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+        (11): BlockWithMasking(
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (drop_path): DropPath(drop_prob=0.100)
+          (norm_1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=768, out_features=3072, bias=True)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=3072, out_features=768, bias=True)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+          (norm_2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+        )
+      )
+      (post_transformer_layer): EinOpsRearrange()
+    )
+  )
+  (modality_heads): ModuleDict(
+    (vision): Sequential(
+      (0): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
+      (1): SelectElement()
+      (2): Linear(in_features=1280, out_features=1024, bias=False)
+    )
+    (audio): Sequential(
+      (0): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
+      (1): SelectElement()
+      (2): Linear(in_features=768, out_features=1024, bias=False)
+    )
+  )
+  (modality_postprocessors): ModuleDict(
+    (vision): Normalize()
+    (audio): Sequential(
+      (0): Normalize()
+      (1): LearnableLogitScaling(logit_scale_init=20.0,learnable=False, max_logit_scale=100)
+    )
+  )
+)
+```