add lora weights

README.md

@@ -0,0 +1,9 @@
+---
+library_name: peft
+---
+## Training procedure
+
+### Framework versions
+
+
+- PEFT 0.5.0

adapter_config.json

@@ -0,0 +1,21 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 8,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 2,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1dd332277b42d4ff7fa716a6e7e816de05005e31c3e4d41f26a59ea53dab5f
+size 4240077

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f74e98de849e2e46e012f58eb09be2a1e7d62439f271a4671d83b54963955790
+size 2205381

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3da0b1f1c9c5463143708574c5432bdfa00894b71e5f01973ff5a68ac6736b25
+size 14575

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adc80dc77a8c026eabfa151ded706e57803f9841e8645d86b11a50ac83423b09
+size 627

trainer_state.json

@@ -0,0 +1,259 @@
+{
+  "best_metric": 0.49536439776420593,
+  "best_model_checkpoint": "output/llama2-7b-chat-hf/checkpoint-12000",
+  "epoch": 1.8354236769654328,
+  "eval_steps": 1000,
+  "global_step": 12000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.923524013459774e-05,
+      "loss": 0.9566,
+      "step": 500
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.8470480269195475e-05,
+      "loss": 0.2966,
+      "step": 1000
+    },
+    {
+      "epoch": 0.15,
+      "eval_loss": 0.5167073011398315,
+      "eval_runtime": 38.9684,
+      "eval_samples_per_second": 12.831,
+      "eval_steps_per_second": 1.617,
+      "step": 1000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.770572040379321e-05,
+      "loss": 0.2828,
+      "step": 1500
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.694096053839095e-05,
+      "loss": 0.2869,
+      "step": 2000
+    },
+    {
+      "epoch": 0.31,
+      "eval_loss": 0.5082407593727112,
+      "eval_runtime": 38.9667,
+      "eval_samples_per_second": 12.831,
+      "eval_steps_per_second": 1.617,
+      "step": 2000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 4.617620067298868e-05,
+      "loss": 0.2855,
+      "step": 2500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.541144080758642e-05,
+      "loss": 0.2754,
+      "step": 3000
+    },
+    {
+      "epoch": 0.46,
+      "eval_loss": 0.5053573250770569,
+      "eval_runtime": 38.968,
+      "eval_samples_per_second": 12.831,
+      "eval_steps_per_second": 1.617,
+      "step": 3000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.4646680942184155e-05,
+      "loss": 0.2909,
+      "step": 3500
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 4.388192107678189e-05,
+      "loss": 0.2904,
+      "step": 4000
+    },
+    {
+      "epoch": 0.61,
+      "eval_loss": 0.5021488070487976,
+      "eval_runtime": 38.9639,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 4000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.311716121137963e-05,
+      "loss": 0.2871,
+      "step": 4500
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.2352401345977364e-05,
+      "loss": 0.2837,
+      "step": 5000
+    },
+    {
+      "epoch": 0.76,
+      "eval_loss": 0.5003100037574768,
+      "eval_runtime": 38.9665,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 5000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.15876414805751e-05,
+      "loss": 0.2775,
+      "step": 5500
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0822881615172836e-05,
+      "loss": 0.2842,
+      "step": 6000
+    },
+    {
+      "epoch": 0.92,
+      "eval_loss": 0.5002090334892273,
+      "eval_runtime": 38.9654,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 6000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.005812174977057e-05,
+      "loss": 0.29,
+      "step": 6500
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.9293361884368315e-05,
+      "loss": 0.2851,
+      "step": 7000
+    },
+    {
+      "epoch": 1.07,
+      "eval_loss": 0.49814751744270325,
+      "eval_runtime": 38.9615,
+      "eval_samples_per_second": 12.833,
+      "eval_steps_per_second": 1.617,
+      "step": 7000
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.8528602018966045e-05,
+      "loss": 0.2758,
+      "step": 7500
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 3.776384215356378e-05,
+      "loss": 0.2763,
+      "step": 8000
+    },
+    {
+      "epoch": 1.22,
+      "eval_loss": 0.498801052570343,
+      "eval_runtime": 38.9636,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 3.699908228816152e-05,
+      "loss": 0.2826,
+      "step": 8500
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 3.623432242275925e-05,
+      "loss": 0.2712,
+      "step": 9000
+    },
+    {
+      "epoch": 1.38,
+      "eval_loss": 0.49798327684402466,
+      "eval_runtime": 38.9634,
+      "eval_samples_per_second": 12.833,
+      "eval_steps_per_second": 1.617,
+      "step": 9000
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 3.546956255735699e-05,
+      "loss": 0.2775,
+      "step": 9500
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 3.470480269195473e-05,
+      "loss": 0.2727,
+      "step": 10000
+    },
+    {
+      "epoch": 1.53,
+      "eval_loss": 0.49744608998298645,
+      "eval_runtime": 38.9669,
+      "eval_samples_per_second": 12.831,
+      "eval_steps_per_second": 1.617,
+      "step": 10000
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 3.394004282655247e-05,
+      "loss": 0.2747,
+      "step": 10500
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 3.31752829611502e-05,
+      "loss": 0.2711,
+      "step": 11000
+    },
+    {
+      "epoch": 1.68,
+      "eval_loss": 0.49606311321258545,
+      "eval_runtime": 38.9636,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 11000
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 3.2410523095747934e-05,
+      "loss": 0.2766,
+      "step": 11500
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 3.164576323034567e-05,
+      "loss": 0.2785,
+      "step": 12000
+    },
+    {
+      "epoch": 1.84,
+      "eval_loss": 0.49536439776420593,
+      "eval_runtime": 38.9645,
+      "eval_samples_per_second": 12.832,
+      "eval_steps_per_second": 1.617,
+      "step": 12000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 32690,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "total_flos": 2.1886994939904e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c45f7ee0d2646c66609d68351301a58ddac962429a216a6cfb609029cad077c4
+size 4027