{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 3.760205659470345,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.9944963455200195,
      "logits/rejected": -2.9324326515197754,
      "logps/chosen": -1.0260854959487915,
      "logps/rejected": -1.189737319946289,
      "loss": -0.0244,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.0260854959487915,
      "rewards/margins": 0.1636517494916916,
      "rewards/rejected": -1.189737319946289,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 4.14164094224387,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -3.0354819297790527,
      "logits/rejected": -2.9778807163238525,
      "logps/chosen": -1.1017096042633057,
      "logps/rejected": -1.1320788860321045,
      "loss": -0.0273,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.1017096042633057,
      "rewards/margins": 0.030369168147444725,
      "rewards/rejected": -1.1320788860321045,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 4.393103002918313,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -3.0005249977111816,
      "logits/rejected": -2.995767116546631,
      "logps/chosen": -1.0664865970611572,
      "logps/rejected": -1.0592315196990967,
      "loss": -0.0311,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.0664865970611572,
      "rewards/margins": -0.007255014963448048,
      "rewards/rejected": -1.0592315196990967,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 4.089316161372408,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -3.015296697616577,
      "logits/rejected": -2.9769978523254395,
      "logps/chosen": -1.0041254758834839,
      "logps/rejected": -1.0410144329071045,
      "loss": -0.0327,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.0041254758834839,
      "rewards/margins": 0.03688892349600792,
      "rewards/rejected": -1.0410144329071045,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 4.216563165990877,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -3.0237412452697754,
      "logits/rejected": -2.973569631576538,
      "logps/chosen": -0.9541431665420532,
      "logps/rejected": -1.1942131519317627,
      "loss": -0.0413,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9541431665420532,
      "rewards/margins": 0.2400699108839035,
      "rewards/rejected": -1.1942131519317627,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 5.658050284469291,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -3.030153274536133,
      "logits/rejected": -2.9625256061553955,
      "logps/chosen": -0.8760479092597961,
      "logps/rejected": -0.9085603952407837,
      "loss": -0.0368,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.8760479092597961,
      "rewards/margins": 0.032512493431568146,
      "rewards/rejected": -0.9085603952407837,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 5.044506012240391,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -3.0141396522521973,
      "logits/rejected": -2.995772361755371,
      "logps/chosen": -0.8659467697143555,
      "logps/rejected": -1.1117627620697021,
      "loss": -0.0476,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8659467697143555,
      "rewards/margins": 0.24581606686115265,
      "rewards/rejected": -1.1117627620697021,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 4.594342778483098,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.9778103828430176,
      "logits/rejected": -2.9474072456359863,
      "logps/chosen": -0.8786975741386414,
      "logps/rejected": -0.9444503784179688,
      "loss": -0.0484,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8786975741386414,
      "rewards/margins": 0.06575289368629456,
      "rewards/rejected": -0.9444503784179688,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 4.865607139601026,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -3.025959014892578,
      "logits/rejected": -3.019007444381714,
      "logps/chosen": -0.8325697183609009,
      "logps/rejected": -0.9673300981521606,
      "loss": -0.0615,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8325697183609009,
      "rewards/margins": 0.134760320186615,
      "rewards/rejected": -0.9673300981521606,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 5.424026197216979,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -2.8837554454803467,
      "logits/rejected": -2.9218649864196777,
      "logps/chosen": -0.9015443921089172,
      "logps/rejected": -1.0318125486373901,
      "loss": -0.0525,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9015443921089172,
      "rewards/margins": 0.13026830554008484,
      "rewards/rejected": -1.0318125486373901,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 7.570587115119674,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -3.0432026386260986,
      "logits/rejected": -3.038170576095581,
      "logps/chosen": -0.9203222393989563,
      "logps/rejected": -1.0882291793823242,
      "loss": -0.0483,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.9203222393989563,
      "rewards/margins": 0.1679070144891739,
      "rewards/rejected": -1.0882291793823242,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 8.483337894809958,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.9748623371124268,
      "logits/rejected": -2.9038889408111572,
      "logps/chosen": -0.9680806398391724,
      "logps/rejected": -1.1494948863983154,
      "loss": -0.0682,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9680806398391724,
      "rewards/margins": 0.18141421675682068,
      "rewards/rejected": -1.1494948863983154,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 6.597482381521931,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -2.9652066230773926,
      "logits/rejected": -2.988760232925415,
      "logps/chosen": -0.9523311853408813,
      "logps/rejected": -1.1883153915405273,
      "loss": -0.0605,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9523311853408813,
      "rewards/margins": 0.23598428070545197,
      "rewards/rejected": -1.1883153915405273,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 7.227660542967968,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -3.0324559211730957,
      "logits/rejected": -2.9957118034362793,
      "logps/chosen": -1.03068208694458,
      "logps/rejected": -1.2021524906158447,
      "loss": -0.0806,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.03068208694458,
      "rewards/margins": 0.1714704930782318,
      "rewards/rejected": -1.2021524906158447,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 8.630957672516889,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.99220609664917,
      "logits/rejected": -2.9557840824127197,
      "logps/chosen": -1.0696258544921875,
      "logps/rejected": -1.317363977432251,
      "loss": -0.0631,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.0696258544921875,
      "rewards/margins": 0.2477382868528366,
      "rewards/rejected": -1.317363977432251,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 6.399531931157647,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -2.994518280029297,
      "logits/rejected": -2.961357831954956,
      "logps/chosen": -1.0338717699050903,
      "logps/rejected": -1.4787652492523193,
      "loss": -0.0783,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0338717699050903,
      "rewards/margins": 0.44489365816116333,
      "rewards/rejected": -1.4787652492523193,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 7.358699467281965,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -2.9917774200439453,
      "logits/rejected": -2.9551658630371094,
      "logps/chosen": -0.9378916025161743,
      "logps/rejected": -1.334387183189392,
      "loss": -0.0833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9378916025161743,
      "rewards/margins": 0.396495521068573,
      "rewards/rejected": -1.334387183189392,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 5.13038386551715,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.9607720375061035,
      "logits/rejected": -2.9549806118011475,
      "logps/chosen": -1.0484223365783691,
      "logps/rejected": -1.3985503911972046,
      "loss": -0.0862,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0484223365783691,
      "rewards/margins": 0.3501282036304474,
      "rewards/rejected": -1.3985503911972046,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 6.228634016037295,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -3.039447784423828,
      "logits/rejected": -3.0229854583740234,
      "logps/chosen": -1.0390279293060303,
      "logps/rejected": -1.2659882307052612,
      "loss": -0.0748,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.0390279293060303,
      "rewards/margins": 0.22696030139923096,
      "rewards/rejected": -1.2659882307052612,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 13.46485741296935,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -3.0565896034240723,
      "logits/rejected": -3.0682952404022217,
      "logps/chosen": -1.0358636379241943,
      "logps/rejected": -1.3846577405929565,
      "loss": -0.0825,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.0358636379241943,
      "rewards/margins": 0.34879425168037415,
      "rewards/rejected": -1.3846577405929565,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 13.308269094275541,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -3.013173818588257,
      "logits/rejected": -3.020575761795044,
      "logps/chosen": -1.2395133972167969,
      "logps/rejected": -1.6027743816375732,
      "loss": -0.1058,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2395133972167969,
      "rewards/margins": 0.363260954618454,
      "rewards/rejected": -1.6027743816375732,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 14.80943809386291,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -2.907844066619873,
      "logits/rejected": -2.9418551921844482,
      "logps/chosen": -1.0689195394515991,
      "logps/rejected": -1.62472403049469,
      "loss": -0.1078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0689195394515991,
      "rewards/margins": 0.5558044910430908,
      "rewards/rejected": -1.62472403049469,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 7.302061093053783,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -3.064298152923584,
      "logits/rejected": -3.0547451972961426,
      "logps/chosen": -1.0759824514389038,
      "logps/rejected": -1.38835608959198,
      "loss": -0.089,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0759824514389038,
      "rewards/margins": 0.3123735785484314,
      "rewards/rejected": -1.38835608959198,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 6.576904824772524,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.995882272720337,
      "logits/rejected": -2.9872708320617676,
      "logps/chosen": -1.010962724685669,
      "logps/rejected": -1.5580971240997314,
      "loss": -0.0761,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.010962724685669,
      "rewards/margins": 0.547134518623352,
      "rewards/rejected": -1.5580971240997314,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 10.915099417857844,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -2.8915393352508545,
      "logits/rejected": -2.841787338256836,
      "logps/chosen": -1.0743449926376343,
      "logps/rejected": -1.400960922241211,
      "loss": -0.0844,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.0743449926376343,
      "rewards/margins": 0.3266158401966095,
      "rewards/rejected": -1.400960922241211,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 7.5813693492052066,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -2.9785983562469482,
      "logits/rejected": -3.023461103439331,
      "logps/chosen": -1.0497643947601318,
      "logps/rejected": -1.5954610109329224,
      "loss": -0.0988,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0497643947601318,
      "rewards/margins": 0.5456965565681458,
      "rewards/rejected": -1.5954610109329224,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 7.880456151156036,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -2.9451870918273926,
      "logits/rejected": -2.9251372814178467,
      "logps/chosen": -1.1994966268539429,
      "logps/rejected": -1.6415138244628906,
      "loss": -0.0959,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1994966268539429,
      "rewards/margins": 0.442017138004303,
      "rewards/rejected": -1.6415138244628906,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 11.389751848981007,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -2.9757840633392334,
      "logits/rejected": -2.9875569343566895,
      "logps/chosen": -1.3223577737808228,
      "logps/rejected": -1.7084649801254272,
      "loss": -0.0795,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3223577737808228,
      "rewards/margins": 0.38610729575157166,
      "rewards/rejected": -1.7084649801254272,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 12.240889169213316,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -2.884559392929077,
      "logits/rejected": -2.9181649684906006,
      "logps/chosen": -1.162393569946289,
      "logps/rejected": -1.6213098764419556,
      "loss": -0.0999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.162393569946289,
      "rewards/margins": 0.4589163362979889,
      "rewards/rejected": -1.6213098764419556,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 13.394133386548184,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.988391160964966,
      "logits/rejected": -2.9592318534851074,
      "logps/chosen": -1.1906192302703857,
      "logps/rejected": -2.017512559890747,
      "loss": -0.0867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1906192302703857,
      "rewards/margins": 0.8268932104110718,
      "rewards/rejected": -2.017512559890747,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 10.341335190859326,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -2.871455192565918,
      "logits/rejected": -2.853935718536377,
      "logps/chosen": -1.3897101879119873,
      "logps/rejected": -1.8974939584732056,
      "loss": -0.1116,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3897101879119873,
      "rewards/margins": 0.5077837705612183,
      "rewards/rejected": -1.8974939584732056,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 14.583251944991732,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -2.824887752532959,
      "logits/rejected": -2.8534679412841797,
      "logps/chosen": -1.3409488201141357,
      "logps/rejected": -2.2745461463928223,
      "loss": -0.1046,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3409488201141357,
      "rewards/margins": 0.933597207069397,
      "rewards/rejected": -2.2745461463928223,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 13.977236800680524,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.9754512310028076,
      "logits/rejected": -2.960723400115967,
      "logps/chosen": -1.255544662475586,
      "logps/rejected": -1.7897508144378662,
      "loss": -0.1166,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.255544662475586,
      "rewards/margins": 0.534206211566925,
      "rewards/rejected": -1.7897508144378662,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 21.73429928886912,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -2.92985200881958,
      "logits/rejected": -2.9375405311584473,
      "logps/chosen": -1.1773474216461182,
      "logps/rejected": -1.8664484024047852,
      "loss": -0.1043,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1773474216461182,
      "rewards/margins": 0.6891010999679565,
      "rewards/rejected": -1.8664484024047852,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 65.37359700720873,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -2.851064682006836,
      "logits/rejected": -2.87498140335083,
      "logps/chosen": -1.0109388828277588,
      "logps/rejected": -1.6845824718475342,
      "loss": -0.1099,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0109388828277588,
      "rewards/margins": 0.6736436486244202,
      "rewards/rejected": -1.6845824718475342,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 18.528930954008196,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.8931515216827393,
      "logits/rejected": -2.873427629470825,
      "logps/chosen": -1.1016333103179932,
      "logps/rejected": -1.9188123941421509,
      "loss": -0.1115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1016333103179932,
      "rewards/margins": 0.8171789050102234,
      "rewards/rejected": -1.9188123941421509,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 7.80304044080231,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -2.8918251991271973,
      "logits/rejected": -2.9344098567962646,
      "logps/chosen": -1.0134468078613281,
      "logps/rejected": -1.6890709400177002,
      "loss": -0.1128,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0134468078613281,
      "rewards/margins": 0.6756240129470825,
      "rewards/rejected": -1.6890709400177002,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 7.7047015056100046,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -2.8743629455566406,
      "logits/rejected": -2.911881685256958,
      "logps/chosen": -1.0063416957855225,
      "logps/rejected": -1.913240671157837,
      "loss": -0.1187,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0063416957855225,
      "rewards/margins": 0.9068989753723145,
      "rewards/rejected": -1.913240671157837,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 7.811346817640016,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -2.9131577014923096,
      "logits/rejected": -2.89136004447937,
      "logps/chosen": -1.1495698690414429,
      "logps/rejected": -2.071730375289917,
      "loss": -0.0945,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1495698690414429,
      "rewards/margins": 0.9221606254577637,
      "rewards/rejected": -2.071730375289917,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 11.061536990800207,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -2.813307523727417,
      "logits/rejected": -2.7539114952087402,
      "logps/chosen": -1.268898367881775,
      "logps/rejected": -1.69308602809906,
      "loss": -0.1163,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.268898367881775,
      "rewards/margins": 0.4241875112056732,
      "rewards/rejected": -1.69308602809906,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 15.040971267646988,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -2.8270034790039062,
      "logits/rejected": -2.855402946472168,
      "logps/chosen": -1.5387921333312988,
      "logps/rejected": -2.168403148651123,
      "loss": -0.1198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5387921333312988,
      "rewards/margins": 0.6296108961105347,
      "rewards/rejected": -2.168403148651123,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 18.3545573133183,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.818117618560791,
      "logits/rejected": -2.7833092212677,
      "logps/chosen": -1.293579339981079,
      "logps/rejected": -1.8654571771621704,
      "loss": -0.1002,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.293579339981079,
      "rewards/margins": 0.5718778371810913,
      "rewards/rejected": -1.8654571771621704,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 8.458661335807625,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -2.7233188152313232,
      "logits/rejected": -2.727560043334961,
      "logps/chosen": -1.3122286796569824,
      "logps/rejected": -2.4676425457000732,
      "loss": -0.1053,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3122286796569824,
      "rewards/margins": 1.1554138660430908,
      "rewards/rejected": -2.4676425457000732,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 8.86883909656063,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -2.719998359680176,
      "logits/rejected": -2.7265241146087646,
      "logps/chosen": -1.129214882850647,
      "logps/rejected": -1.7588284015655518,
      "loss": -0.1146,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.129214882850647,
      "rewards/margins": 0.6296136975288391,
      "rewards/rejected": -1.7588284015655518,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 10.016548371714508,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -2.811687707901001,
      "logits/rejected": -2.7085366249084473,
      "logps/chosen": -1.3638594150543213,
      "logps/rejected": -2.3513691425323486,
      "loss": -0.1184,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3638594150543213,
      "rewards/margins": 0.9875099062919617,
      "rewards/rejected": -2.3513691425323486,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 21.998434755048503,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -2.6830623149871826,
      "logits/rejected": -2.6364707946777344,
      "logps/chosen": -1.3768701553344727,
      "logps/rejected": -2.424539089202881,
      "loss": -0.1267,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3768701553344727,
      "rewards/margins": 1.0476689338684082,
      "rewards/rejected": -2.424539089202881,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 15.145805829963615,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -2.4627432823181152,
      "logits/rejected": -2.4076457023620605,
      "logps/chosen": -1.2042968273162842,
      "logps/rejected": -2.078500270843506,
      "loss": -0.1185,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.2042968273162842,
      "rewards/margins": 0.8742035031318665,
      "rewards/rejected": -2.078500270843506,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 10.783814010063464,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.5319347381591797,
      "logits/rejected": -2.5071659088134766,
      "logps/chosen": -1.5030553340911865,
      "logps/rejected": -2.248628616333008,
      "loss": -0.1178,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5030553340911865,
      "rewards/margins": 0.7455734014511108,
      "rewards/rejected": -2.248628616333008,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 17.35265824839872,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -2.629790782928467,
      "logits/rejected": -2.5811970233917236,
      "logps/chosen": -1.1765772104263306,
      "logps/rejected": -2.45619535446167,
      "loss": -0.1283,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1765772104263306,
      "rewards/margins": 1.2796180248260498,
      "rewards/rejected": -2.45619535446167,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 21.631571252123923,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -2.559217929840088,
      "logits/rejected": -2.4746146202087402,
      "logps/chosen": -1.4234548807144165,
      "logps/rejected": -2.4785306453704834,
      "loss": -0.1032,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4234548807144165,
      "rewards/margins": 1.055075764656067,
      "rewards/rejected": -2.4785306453704834,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 10.665899447598967,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -2.662745237350464,
      "logits/rejected": -2.6075425148010254,
      "logps/chosen": -1.1387856006622314,
      "logps/rejected": -2.129990577697754,
      "loss": -0.1171,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.1387856006622314,
      "rewards/margins": 0.9912050366401672,
      "rewards/rejected": -2.129990577697754,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 30.24510546028332,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -2.5815982818603516,
      "logits/rejected": -2.5034749507904053,
      "logps/chosen": -1.3083416223526,
      "logps/rejected": -2.3620128631591797,
      "loss": -0.1357,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3083416223526,
      "rewards/margins": 1.0536713600158691,
      "rewards/rejected": -2.3620128631591797,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 36.43768779189584,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -2.4912540912628174,
      "logits/rejected": -2.503256320953369,
      "logps/chosen": -1.527940034866333,
      "logps/rejected": -2.379206895828247,
      "loss": -0.1328,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.527940034866333,
      "rewards/margins": 0.8512667417526245,
      "rewards/rejected": -2.379206895828247,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 21.411949693790206,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.429790735244751,
      "logits/rejected": -2.451469898223877,
      "logps/chosen": -1.8636459112167358,
      "logps/rejected": -2.370575428009033,
      "loss": -0.1231,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.8636459112167358,
      "rewards/margins": 0.5069290399551392,
      "rewards/rejected": -2.370575428009033,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 26.28588094272757,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -2.484380006790161,
      "logits/rejected": -2.3554978370666504,
      "logps/chosen": -1.374985933303833,
      "logps/rejected": -3.2982125282287598,
      "loss": -0.1605,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.374985933303833,
      "rewards/margins": 1.9232263565063477,
      "rewards/rejected": -3.2982125282287598,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 15.64313913527933,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -2.5084352493286133,
      "logits/rejected": -2.379485607147217,
      "logps/chosen": -1.2756072282791138,
      "logps/rejected": -2.862361431121826,
      "loss": -0.1283,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2756072282791138,
      "rewards/margins": 1.586754560470581,
      "rewards/rejected": -2.862361431121826,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 31.160163808849045,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -2.464437484741211,
      "logits/rejected": -2.4234752655029297,
      "logps/chosen": -1.7923996448516846,
      "logps/rejected": -3.0464653968811035,
      "loss": -0.1107,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7923996448516846,
      "rewards/margins": 1.2540655136108398,
      "rewards/rejected": -3.0464653968811035,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 14.525714466074616,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -2.3748879432678223,
      "logits/rejected": -2.277477741241455,
      "logps/chosen": -1.8306829929351807,
      "logps/rejected": -2.5916666984558105,
      "loss": -0.1171,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8306829929351807,
      "rewards/margins": 0.7609835863113403,
      "rewards/rejected": -2.5916666984558105,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 12.205724246076834,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -2.5410079956054688,
      "logits/rejected": -2.368896484375,
      "logps/chosen": -1.3447208404541016,
      "logps/rejected": -2.198944091796875,
      "loss": -0.1286,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3447208404541016,
      "rewards/margins": 0.8542232513427734,
      "rewards/rejected": -2.198944091796875,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 13.048722996544631,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -2.3360071182250977,
      "logits/rejected": -2.238037347793579,
      "logps/chosen": -1.3523575067520142,
      "logps/rejected": -2.568472146987915,
      "loss": -0.1212,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3523575067520142,
      "rewards/margins": 1.2161142826080322,
      "rewards/rejected": -2.568472146987915,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 14.965881571164392,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -2.4726943969726562,
      "logits/rejected": -2.4096648693084717,
      "logps/chosen": -1.4510114192962646,
      "logps/rejected": -2.0148894786834717,
      "loss": -0.1379,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4510114192962646,
      "rewards/margins": 0.563878059387207,
      "rewards/rejected": -2.0148894786834717,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 14.196438929994413,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -2.434021472930908,
      "logits/rejected": -2.2209062576293945,
      "logps/chosen": -1.2371381521224976,
      "logps/rejected": -2.5744917392730713,
      "loss": -0.1344,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2371381521224976,
      "rewards/margins": 1.3373535871505737,
      "rewards/rejected": -2.5744917392730713,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 13.34734425640694,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -2.482234477996826,
      "logits/rejected": -2.3337721824645996,
      "logps/chosen": -1.7002273797988892,
      "logps/rejected": -2.32104754447937,
      "loss": -0.1352,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7002273797988892,
      "rewards/margins": 0.6208202838897705,
      "rewards/rejected": -2.32104754447937,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 54.856648124460555,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -2.4059951305389404,
      "logits/rejected": -2.2941792011260986,
      "logps/chosen": -1.7276633977890015,
      "logps/rejected": -2.791877031326294,
      "loss": -0.1395,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.7276633977890015,
      "rewards/margins": 1.0642132759094238,
      "rewards/rejected": -2.791877031326294,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 16.338313868801983,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -2.3474369049072266,
      "logits/rejected": -2.2156565189361572,
      "logps/chosen": -1.581926941871643,
      "logps/rejected": -3.4111123085021973,
      "loss": -0.1119,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.581926941871643,
      "rewards/margins": 1.8291854858398438,
      "rewards/rejected": -3.4111123085021973,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 27.599650373019013,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -2.424649238586426,
      "logits/rejected": -2.313764810562134,
      "logps/chosen": -1.4561196565628052,
      "logps/rejected": -2.5569052696228027,
      "loss": -0.1255,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4561196565628052,
      "rewards/margins": 1.1007858514785767,
      "rewards/rejected": -2.5569052696228027,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 13.71174939782389,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -2.182265520095825,
      "logits/rejected": -2.240220069885254,
      "logps/chosen": -2.133391857147217,
      "logps/rejected": -2.705057382583618,
      "loss": -0.1229,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.133391857147217,
      "rewards/margins": 0.571665346622467,
      "rewards/rejected": -2.705057382583618,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 22.094952760541535,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -2.351863384246826,
      "logits/rejected": -2.2399449348449707,
      "logps/chosen": -1.3272682428359985,
      "logps/rejected": -3.206008195877075,
      "loss": -0.1347,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3272682428359985,
      "rewards/margins": 1.8787400722503662,
      "rewards/rejected": -3.206008195877075,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 14.410515050019907,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -2.3990814685821533,
      "logits/rejected": -2.256584644317627,
      "logps/chosen": -1.4144983291625977,
      "logps/rejected": -2.471238851547241,
      "loss": -0.1205,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4144983291625977,
      "rewards/margins": 1.056740641593933,
      "rewards/rejected": -2.471238851547241,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 17.737495980302032,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -2.3040690422058105,
      "logits/rejected": -1.9883592128753662,
      "logps/chosen": -1.8120450973510742,
      "logps/rejected": -2.608847141265869,
      "loss": -0.1314,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.8120450973510742,
      "rewards/margins": 0.7968018651008606,
      "rewards/rejected": -2.608847141265869,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 147.24723390344812,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -2.2860684394836426,
      "logits/rejected": -2.1697564125061035,
      "logps/chosen": -1.6789791584014893,
      "logps/rejected": -3.0929505825042725,
      "loss": -0.1132,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6789791584014893,
      "rewards/margins": 1.413971185684204,
      "rewards/rejected": -3.0929505825042725,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 25.64073118103101,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -2.218724012374878,
      "logits/rejected": -2.1296398639678955,
      "logps/chosen": -1.6159321069717407,
      "logps/rejected": -2.5099380016326904,
      "loss": -0.1222,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6159321069717407,
      "rewards/margins": 0.8940057754516602,
      "rewards/rejected": -2.5099380016326904,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 12.902928067743126,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -2.3579747676849365,
      "logits/rejected": -2.1200146675109863,
      "logps/chosen": -1.5285216569900513,
      "logps/rejected": -3.1276838779449463,
      "loss": -0.145,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5285216569900513,
      "rewards/margins": 1.5991623401641846,
      "rewards/rejected": -3.1276838779449463,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 15.785906225773893,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -2.407064914703369,
      "logits/rejected": -2.30210542678833,
      "logps/chosen": -1.3575342893600464,
      "logps/rejected": -2.2857096195220947,
      "loss": -0.1468,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.3575342893600464,
      "rewards/margins": 0.9281753301620483,
      "rewards/rejected": -2.2857096195220947,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 66.73212890394835,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -2.2724781036376953,
      "logits/rejected": -2.2895565032958984,
      "logps/chosen": -1.549403429031372,
      "logps/rejected": -2.9475514888763428,
      "loss": -0.1203,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.549403429031372,
      "rewards/margins": 1.3981479406356812,
      "rewards/rejected": -2.9475514888763428,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 38.70646418643775,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -2.399785041809082,
      "logits/rejected": -2.3085193634033203,
      "logps/chosen": -1.3037949800491333,
      "logps/rejected": -2.2208690643310547,
      "loss": -0.1115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3037949800491333,
      "rewards/margins": 0.9170739054679871,
      "rewards/rejected": -2.2208690643310547,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 85.36661297405541,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -2.350160598754883,
      "logits/rejected": -2.304274559020996,
      "logps/chosen": -1.328881859779358,
      "logps/rejected": -2.426609516143799,
      "loss": -0.1355,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.328881859779358,
      "rewards/margins": 1.0977277755737305,
      "rewards/rejected": -2.426609516143799,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 21.914030829763547,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -2.41226863861084,
      "logits/rejected": -2.258676767349243,
      "logps/chosen": -1.3917608261108398,
      "logps/rejected": -2.255201578140259,
      "loss": -0.1082,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3917608261108398,
      "rewards/margins": 0.8634408712387085,
      "rewards/rejected": -2.255201578140259,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 11.128742517489252,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -2.3416173458099365,
      "logits/rejected": -2.183840274810791,
      "logps/chosen": -1.4819704294204712,
      "logps/rejected": -2.7424230575561523,
      "loss": -0.1199,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4819704294204712,
      "rewards/margins": 1.2604527473449707,
      "rewards/rejected": -2.7424230575561523,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 13.499128871751621,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -2.4478988647460938,
      "logits/rejected": -2.2530007362365723,
      "logps/chosen": -1.503173589706421,
      "logps/rejected": -2.4205775260925293,
      "loss": -0.1196,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.503173589706421,
      "rewards/margins": 0.9174041748046875,
      "rewards/rejected": -2.4205775260925293,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 46.00477049565933,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -2.428011894226074,
      "logits/rejected": -2.195507526397705,
      "logps/chosen": -1.488608956336975,
      "logps/rejected": -3.2226758003234863,
      "loss": -0.1329,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.488608956336975,
      "rewards/margins": 1.7340667247772217,
      "rewards/rejected": -3.2226758003234863,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 15.147104463873568,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -2.3303403854370117,
      "logits/rejected": -2.2017440795898438,
      "logps/chosen": -1.4811475276947021,
      "logps/rejected": -3.1947004795074463,
      "loss": -0.1345,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4811475276947021,
      "rewards/margins": 1.7135528326034546,
      "rewards/rejected": -3.1947004795074463,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 32.54043568171796,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -2.3127968311309814,
      "logits/rejected": -2.2588541507720947,
      "logps/chosen": -1.7812035083770752,
      "logps/rejected": -2.363581418991089,
      "loss": -0.1144,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7812035083770752,
      "rewards/margins": 0.5823777318000793,
      "rewards/rejected": -2.363581418991089,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 16.364361772449453,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -2.2632575035095215,
      "logits/rejected": -2.1989002227783203,
      "logps/chosen": -1.3589812517166138,
      "logps/rejected": -2.665706157684326,
      "loss": -0.1402,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3589812517166138,
      "rewards/margins": 1.3067247867584229,
      "rewards/rejected": -2.665706157684326,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 24.459898730148318,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -2.4754738807678223,
      "logits/rejected": -2.2798328399658203,
      "logps/chosen": -1.1175788640975952,
      "logps/rejected": -2.699981212615967,
      "loss": -0.1488,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1175788640975952,
      "rewards/margins": 1.5824021100997925,
      "rewards/rejected": -2.699981212615967,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 20.15635024127512,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -2.3345370292663574,
      "logits/rejected": -2.0842413902282715,
      "logps/chosen": -1.416595458984375,
      "logps/rejected": -2.772968053817749,
      "loss": -0.1388,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.416595458984375,
      "rewards/margins": 1.3563728332519531,
      "rewards/rejected": -2.772968053817749,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 25.1930677516597,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -2.2844507694244385,
      "logits/rejected": -2.2151055335998535,
      "logps/chosen": -1.5157394409179688,
      "logps/rejected": -2.6189606189727783,
      "loss": -0.1335,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5157394409179688,
      "rewards/margins": 1.1032211780548096,
      "rewards/rejected": -2.6189606189727783,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 94.3340138658481,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -2.3165175914764404,
      "logits/rejected": -2.216142177581787,
      "logps/chosen": -1.6333303451538086,
      "logps/rejected": -2.6600871086120605,
      "loss": -0.1365,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6333303451538086,
      "rewards/margins": 1.0267568826675415,
      "rewards/rejected": -2.6600871086120605,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 37.68445074958579,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -2.173781394958496,
      "logits/rejected": -2.0770103931427,
      "logps/chosen": -1.7893623113632202,
      "logps/rejected": -2.782428026199341,
      "loss": -0.1362,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.7893623113632202,
      "rewards/margins": 0.9930654764175415,
      "rewards/rejected": -2.782428026199341,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 65.20343901413754,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -2.4339003562927246,
      "logits/rejected": -2.235538959503174,
      "logps/chosen": -1.1971713304519653,
      "logps/rejected": -2.028782367706299,
      "loss": -0.1358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1971713304519653,
      "rewards/margins": 0.8316110372543335,
      "rewards/rejected": -2.028782367706299,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 29.43315569875579,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -2.406641721725464,
      "logits/rejected": -2.2101848125457764,
      "logps/chosen": -1.7016384601593018,
      "logps/rejected": -2.7031149864196777,
      "loss": -0.1135,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7016384601593018,
      "rewards/margins": 1.0014766454696655,
      "rewards/rejected": -2.7031149864196777,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 58.96075147402077,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -2.5120296478271484,
      "logits/rejected": -2.0890145301818848,
      "logps/chosen": -1.2953853607177734,
      "logps/rejected": -3.554316759109497,
      "loss": -0.1399,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2953853607177734,
      "rewards/margins": 2.2589316368103027,
      "rewards/rejected": -3.554316759109497,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 34.51709660208177,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -2.295931339263916,
      "logits/rejected": -2.02848219871521,
      "logps/chosen": -1.3973679542541504,
      "logps/rejected": -3.212092638015747,
      "loss": -0.145,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3973679542541504,
      "rewards/margins": 1.814724326133728,
      "rewards/rejected": -3.212092638015747,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 11.228558025822675,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -2.295358180999756,
      "logits/rejected": -2.151076316833496,
      "logps/chosen": -1.4753532409667969,
      "logps/rejected": -2.6792643070220947,
      "loss": -0.131,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4753532409667969,
      "rewards/margins": 1.2039110660552979,
      "rewards/rejected": -2.6792643070220947,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 15.547524058275151,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -2.3587679862976074,
      "logits/rejected": -2.1848464012145996,
      "logps/chosen": -1.336055874824524,
      "logps/rejected": -2.380079746246338,
      "loss": -0.1371,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.336055874824524,
      "rewards/margins": 1.044023871421814,
      "rewards/rejected": -2.380079746246338,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": -0.1067421951134,
      "train_runtime": 20026.0763,
      "train_samples_per_second": 3.053,
      "train_steps_per_second": 0.024
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 239,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}