diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,18200 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 7.577480151300404,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.05669214576482773,
+      "logits/rejected": 0.1520434319972992,
+      "logps/chosen": -1.7159297466278076,
+      "logps/rejected": -1.889162302017212,
+      "loss": 1.1644,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.7159297466278076,
+      "rewards/margins": 0.17323248088359833,
+      "rewards/rejected": -1.889162302017212,
+      "semantic_entropy": 0.6584523320198059,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 9.0217422554576,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": -0.003244952065870166,
+      "logits/rejected": 0.11460573971271515,
+      "logps/chosen": -1.802811861038208,
+      "logps/rejected": -1.8454315662384033,
+      "loss": 1.2448,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.802811861038208,
+      "rewards/margins": 0.04261985421180725,
+      "rewards/rejected": -1.8454315662384033,
+      "semantic_entropy": 0.6393086314201355,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 8.728835880859794,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.040742259472608566,
+      "logits/rejected": 0.05471045896410942,
+      "logps/chosen": -1.6357338428497314,
+      "logps/rejected": -1.7656360864639282,
+      "loss": 1.2716,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6357338428497314,
+      "rewards/margins": 0.12990199029445648,
+      "rewards/rejected": -1.7656360864639282,
+      "semantic_entropy": 0.6929879188537598,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 7.849356991910005,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.049366533756256104,
+      "logits/rejected": 0.03429045528173447,
+      "logps/chosen": -1.7241863012313843,
+      "logps/rejected": -1.8041187524795532,
+      "loss": 1.2777,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7241863012313843,
+      "rewards/margins": 0.07993234694004059,
+      "rewards/rejected": -1.8041187524795532,
+      "semantic_entropy": 0.6686957478523254,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 13.137845239437253,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.0431622751057148,
+      "logits/rejected": 0.040008969604969025,
+      "logps/chosen": -1.869818091392517,
+      "logps/rejected": -1.7796188592910767,
+      "loss": 1.4025,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -1.869818091392517,
+      "rewards/margins": -0.09019950777292252,
+      "rewards/rejected": -1.7796188592910767,
+      "semantic_entropy": 0.6431052684783936,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 7.25235385632147,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.09445489943027496,
+      "logits/rejected": -0.0017687976360321045,
+      "logps/chosen": -1.9100134372711182,
+      "logps/rejected": -1.833946943283081,
+      "loss": 1.2903,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -1.9100134372711182,
+      "rewards/margins": -0.07606656849384308,
+      "rewards/rejected": -1.833946943283081,
+      "semantic_entropy": 0.6175903081893921,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 7.884666090378799,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.058491092175245285,
+      "logits/rejected": 0.09979893267154694,
+      "logps/chosen": -1.8480303287506104,
+      "logps/rejected": -1.998309850692749,
+      "loss": 1.2815,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8480303287506104,
+      "rewards/margins": 0.1502797156572342,
+      "rewards/rejected": -1.998309850692749,
+      "semantic_entropy": 0.6345275044441223,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 7.090268481750445,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.024502048268914223,
+      "logits/rejected": 0.1947236806154251,
+      "logps/chosen": -1.881303071975708,
+      "logps/rejected": -1.7437727451324463,
+      "loss": 1.3449,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.881303071975708,
+      "rewards/margins": -0.13753047585487366,
+      "rewards/rejected": -1.7437727451324463,
+      "semantic_entropy": 0.6431247591972351,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 11.972079100941544,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.033025600016117096,
+      "logits/rejected": 0.23134207725524902,
+      "logps/chosen": -1.8372999429702759,
+      "logps/rejected": -1.8718618154525757,
+      "loss": 1.3077,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8372999429702759,
+      "rewards/margins": 0.03456193208694458,
+      "rewards/rejected": -1.8718618154525757,
+      "semantic_entropy": 0.6490525007247925,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 9.800124025385449,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.04672237113118172,
+      "logits/rejected": 0.1039384976029396,
+      "logps/chosen": -1.898736596107483,
+      "logps/rejected": -1.778319001197815,
+      "loss": 1.3485,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.898736596107483,
+      "rewards/margins": -0.12041763216257095,
+      "rewards/rejected": -1.778319001197815,
+      "semantic_entropy": 0.6337752938270569,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 7.141957745461847,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.11001928150653839,
+      "logits/rejected": 0.10958278179168701,
+      "logps/chosen": -1.8353521823883057,
+      "logps/rejected": -1.8687827587127686,
+      "loss": 1.3132,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8353521823883057,
+      "rewards/margins": 0.03343046084046364,
+      "rewards/rejected": -1.8687827587127686,
+      "semantic_entropy": 0.6433704495429993,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 7.70232764426562,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.07812195271253586,
+      "logits/rejected": 0.11321119219064713,
+      "logps/chosen": -1.7913398742675781,
+      "logps/rejected": -1.895944595336914,
+      "loss": 1.2206,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.7913398742675781,
+      "rewards/margins": 0.10460458695888519,
+      "rewards/rejected": -1.895944595336914,
+      "semantic_entropy": 0.63536137342453,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 6.418184455399686,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.028376828879117966,
+      "logits/rejected": 0.11893482506275177,
+      "logps/chosen": -1.640639305114746,
+      "logps/rejected": -1.7711946964263916,
+      "loss": 1.2237,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.640639305114746,
+      "rewards/margins": 0.13055558502674103,
+      "rewards/rejected": -1.7711946964263916,
+      "semantic_entropy": 0.695518970489502,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 11.31270623430761,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.06639423221349716,
+      "logits/rejected": 0.08414672315120697,
+      "logps/chosen": -1.7696411609649658,
+      "logps/rejected": -1.8152275085449219,
+      "loss": 1.3189,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -1.7696411609649658,
+      "rewards/margins": 0.04558655619621277,
+      "rewards/rejected": -1.8152275085449219,
+      "semantic_entropy": 0.6532689332962036,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 11.11601249303447,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.03809754550457001,
+      "logits/rejected": 0.14747920632362366,
+      "logps/chosen": -1.7821757793426514,
+      "logps/rejected": -2.044076681137085,
+      "loss": 1.1742,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.7821757793426514,
+      "rewards/margins": 0.2619008421897888,
+      "rewards/rejected": -2.044076681137085,
+      "semantic_entropy": 0.6323312520980835,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 7.543196712253653,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": 0.02288135699927807,
+      "logits/rejected": 0.12928631901741028,
+      "logps/chosen": -1.7240753173828125,
+      "logps/rejected": -1.757128357887268,
+      "loss": 1.2809,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7240753173828125,
+      "rewards/margins": 0.03305302932858467,
+      "rewards/rejected": -1.757128357887268,
+      "semantic_entropy": 0.6675456762313843,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 6.204260212909473,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.14709827303886414,
+      "logits/rejected": 0.09955728054046631,
+      "logps/chosen": -1.7977319955825806,
+      "logps/rejected": -1.9758974313735962,
+      "loss": 1.2373,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.7977319955825806,
+      "rewards/margins": 0.17816540598869324,
+      "rewards/rejected": -1.9758974313735962,
+      "semantic_entropy": 0.6462455987930298,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 12.76749951189234,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.0821002870798111,
+      "logits/rejected": 0.04157263785600662,
+      "logps/chosen": -1.7592077255249023,
+      "logps/rejected": -1.7844018936157227,
+      "loss": 1.3251,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.7592077255249023,
+      "rewards/margins": 0.025194281712174416,
+      "rewards/rejected": -1.7844018936157227,
+      "semantic_entropy": 0.6664949059486389,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 5.000806601728965,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.059259116649627686,
+      "logits/rejected": 0.08907194435596466,
+      "logps/chosen": -1.8205684423446655,
+      "logps/rejected": -1.9245706796646118,
+      "loss": 1.2591,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.8205684423446655,
+      "rewards/margins": 0.10400215536355972,
+      "rewards/rejected": -1.9245706796646118,
+      "semantic_entropy": 0.6380980610847473,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 7.265568549850801,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.02074522152543068,
+      "logits/rejected": 0.04259929060935974,
+      "logps/chosen": -1.7065422534942627,
+      "logps/rejected": -1.8150056600570679,
+      "loss": 1.2315,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7065422534942627,
+      "rewards/margins": 0.10846362262964249,
+      "rewards/rejected": -1.8150056600570679,
+      "semantic_entropy": 0.6694960594177246,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 7.558006354919823,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.048721201717853546,
+      "logits/rejected": 0.0760502964258194,
+      "logps/chosen": -1.6596462726593018,
+      "logps/rejected": -1.8243238925933838,
+      "loss": 1.2203,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6596462726593018,
+      "rewards/margins": 0.16467776894569397,
+      "rewards/rejected": -1.8243238925933838,
+      "semantic_entropy": 0.6800812482833862,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 7.366195022286922,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": 0.020339591428637505,
+      "logits/rejected": 0.115842305123806,
+      "logps/chosen": -1.7015666961669922,
+      "logps/rejected": -1.7593681812286377,
+      "loss": 1.2898,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7015666961669922,
+      "rewards/margins": 0.05780160427093506,
+      "rewards/rejected": -1.7593681812286377,
+      "semantic_entropy": 0.6733224987983704,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 8.35586481366848,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.039164334535598755,
+      "logits/rejected": 0.24791447818279266,
+      "logps/chosen": -1.6922931671142578,
+      "logps/rejected": -1.9841411113739014,
+      "loss": 1.1465,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6922931671142578,
+      "rewards/margins": 0.2918480634689331,
+      "rewards/rejected": -1.9841411113739014,
+      "semantic_entropy": 0.649268388748169,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 6.08797656371398,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.06477510929107666,
+      "logits/rejected": 0.11136678606271744,
+      "logps/chosen": -1.7872076034545898,
+      "logps/rejected": -1.9132124185562134,
+      "loss": 1.2236,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7872076034545898,
+      "rewards/margins": 0.1260049045085907,
+      "rewards/rejected": -1.9132124185562134,
+      "semantic_entropy": 0.64605712890625,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 8.247036462260532,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.05269402265548706,
+      "logits/rejected": 0.08044368028640747,
+      "logps/chosen": -1.7095565795898438,
+      "logps/rejected": -1.6542142629623413,
+      "loss": 1.3276,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.7095565795898438,
+      "rewards/margins": -0.05534226819872856,
+      "rewards/rejected": -1.6542142629623413,
+      "semantic_entropy": 0.6840522885322571,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 7.788118796235791,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.05479375645518303,
+      "logits/rejected": 0.1922006756067276,
+      "logps/chosen": -1.7594177722930908,
+      "logps/rejected": -1.8804855346679688,
+      "loss": 1.1862,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7594177722930908,
+      "rewards/margins": 0.12106774002313614,
+      "rewards/rejected": -1.8804855346679688,
+      "semantic_entropy": 0.6431999802589417,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 14.353357465298114,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.04986584931612015,
+      "logits/rejected": 0.06863486766815186,
+      "logps/chosen": -1.821009874343872,
+      "logps/rejected": -1.8278930187225342,
+      "loss": 1.3019,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.821009874343872,
+      "rewards/margins": 0.006883461959660053,
+      "rewards/rejected": -1.8278930187225342,
+      "semantic_entropy": 0.6367494463920593,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 11.395034093411931,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.03189806267619133,
+      "logits/rejected": 0.13713029026985168,
+      "logps/chosen": -1.7823625802993774,
+      "logps/rejected": -1.9552139043807983,
+      "loss": 1.1781,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.7823625802993774,
+      "rewards/margins": 0.1728512942790985,
+      "rewards/rejected": -1.9552139043807983,
+      "semantic_entropy": 0.6267226934432983,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 9.546921917695292,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": -0.009675289504230022,
+      "logits/rejected": 0.1468123495578766,
+      "logps/chosen": -1.7033294439315796,
+      "logps/rejected": -1.8265163898468018,
+      "loss": 1.2205,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.7033294439315796,
+      "rewards/margins": 0.12318698316812515,
+      "rewards/rejected": -1.8265163898468018,
+      "semantic_entropy": 0.657718300819397,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 9.998934450431614,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.05009545013308525,
+      "logits/rejected": 0.11495450884103775,
+      "logps/chosen": -1.6608514785766602,
+      "logps/rejected": -1.660271406173706,
+      "loss": 1.3187,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.6608514785766602,
+      "rewards/margins": -0.0005798399215564132,
+      "rewards/rejected": -1.660271406173706,
+      "semantic_entropy": 0.7037652134895325,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 8.438985740613225,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.042882729321718216,
+      "logits/rejected": 0.0062928213737905025,
+      "logps/chosen": -1.6918509006500244,
+      "logps/rejected": -1.7746083736419678,
+      "loss": 1.2443,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.6918509006500244,
+      "rewards/margins": 0.08275739848613739,
+      "rewards/rejected": -1.7746083736419678,
+      "semantic_entropy": 0.6715037822723389,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 6.847224847734815,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.14369115233421326,
+      "logits/rejected": -0.004838567227125168,
+      "logps/chosen": -1.8290367126464844,
+      "logps/rejected": -1.8086496591567993,
+      "loss": 1.3173,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8290367126464844,
+      "rewards/margins": -0.02038700319826603,
+      "rewards/rejected": -1.8086496591567993,
+      "semantic_entropy": 0.6431725025177002,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 11.337214136891104,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.05288670212030411,
+      "logits/rejected": 0.12041208893060684,
+      "logps/chosen": -1.6387144327163696,
+      "logps/rejected": -1.8127260208129883,
+      "loss": 1.2349,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.6387144327163696,
+      "rewards/margins": 0.1740114390850067,
+      "rewards/rejected": -1.8127260208129883,
+      "semantic_entropy": 0.6767231225967407,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 13.456418600616226,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.08759705722332001,
+      "logits/rejected": -0.0350174717605114,
+      "logps/chosen": -1.8107351064682007,
+      "logps/rejected": -1.8460609912872314,
+      "loss": 1.2793,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8107351064682007,
+      "rewards/margins": 0.03532556816935539,
+      "rewards/rejected": -1.8460609912872314,
+      "semantic_entropy": 0.6370847821235657,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 8.511102911280044,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": 0.04904542863368988,
+      "logits/rejected": 0.047991104423999786,
+      "logps/chosen": -1.6801316738128662,
+      "logps/rejected": -1.7776981592178345,
+      "loss": 1.2799,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6801316738128662,
+      "rewards/margins": 0.09756648540496826,
+      "rewards/rejected": -1.7776981592178345,
+      "semantic_entropy": 0.6753233671188354,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 8.438476631150133,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": 0.02723088301718235,
+      "logits/rejected": 0.026967281475663185,
+      "logps/chosen": -1.7155345678329468,
+      "logps/rejected": -1.839882493019104,
+      "loss": 1.26,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.7155345678329468,
+      "rewards/margins": 0.12434802204370499,
+      "rewards/rejected": -1.839882493019104,
+      "semantic_entropy": 0.6664156317710876,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 8.345759071037545,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.11280833184719086,
+      "logits/rejected": -0.02105618081986904,
+      "logps/chosen": -1.671248197555542,
+      "logps/rejected": -1.7454736232757568,
+      "loss": 1.2999,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.671248197555542,
+      "rewards/margins": 0.07422558218240738,
+      "rewards/rejected": -1.7454736232757568,
+      "semantic_entropy": 0.6829355359077454,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 8.284360908015822,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": -0.027448534965515137,
+      "logits/rejected": 0.09939461946487427,
+      "logps/chosen": -1.7798713445663452,
+      "logps/rejected": -1.8399927616119385,
+      "loss": 1.2321,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.7798713445663452,
+      "rewards/margins": 0.06012124568223953,
+      "rewards/rejected": -1.8399927616119385,
+      "semantic_entropy": 0.631985604763031,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 6.411380302367792,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": 0.03862842917442322,
+      "logits/rejected": 0.19834889471530914,
+      "logps/chosen": -1.535895586013794,
+      "logps/rejected": -1.680272102355957,
+      "loss": 1.2313,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.535895586013794,
+      "rewards/margins": 0.14437641203403473,
+      "rewards/rejected": -1.680272102355957,
+      "semantic_entropy": 0.7183574438095093,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 13.025120389771589,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.04185447841882706,
+      "logits/rejected": 0.10197518765926361,
+      "logps/chosen": -1.6903976202011108,
+      "logps/rejected": -1.68011474609375,
+      "loss": 1.291,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6903976202011108,
+      "rewards/margins": -0.010283132083714008,
+      "rewards/rejected": -1.68011474609375,
+      "semantic_entropy": 0.6760302782058716,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 13.720694556779323,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.04812673479318619,
+      "logits/rejected": 0.09947986900806427,
+      "logps/chosen": -1.6579277515411377,
+      "logps/rejected": -1.6668905019760132,
+      "loss": 1.2818,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6579277515411377,
+      "rewards/margins": 0.008962845429778099,
+      "rewards/rejected": -1.6668905019760132,
+      "semantic_entropy": 0.6777267456054688,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 14.45107627792218,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.11372479051351547,
+      "logits/rejected": 0.08760619163513184,
+      "logps/chosen": -1.6680940389633179,
+      "logps/rejected": -1.8717817068099976,
+      "loss": 1.1798,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6680940389633179,
+      "rewards/margins": 0.20368759334087372,
+      "rewards/rejected": -1.8717817068099976,
+      "semantic_entropy": 0.6688157320022583,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 8.950102229959466,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.16349974274635315,
+      "logits/rejected": 0.08879305422306061,
+      "logps/chosen": -1.615807294845581,
+      "logps/rejected": -1.7200053930282593,
+      "loss": 1.2011,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.615807294845581,
+      "rewards/margins": 0.10419821739196777,
+      "rewards/rejected": -1.7200053930282593,
+      "semantic_entropy": 0.6942529082298279,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 17.2958658665997,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": 0.07525455951690674,
+      "logits/rejected": 0.17859020829200745,
+      "logps/chosen": -1.6143310070037842,
+      "logps/rejected": -1.8492858409881592,
+      "loss": 1.1605,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.6143310070037842,
+      "rewards/margins": 0.23495467007160187,
+      "rewards/rejected": -1.8492858409881592,
+      "semantic_entropy": 0.6733145117759705,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 7.055835953874801,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.0779423862695694,
+      "logits/rejected": 0.09895094484090805,
+      "logps/chosen": -1.5765641927719116,
+      "logps/rejected": -1.7272427082061768,
+      "loss": 1.188,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5765641927719116,
+      "rewards/margins": 0.1506786346435547,
+      "rewards/rejected": -1.7272427082061768,
+      "semantic_entropy": 0.6905933618545532,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 6.375048256775752,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": -0.004890531301498413,
+      "logits/rejected": 0.07432994991540909,
+      "logps/chosen": -1.6714308261871338,
+      "logps/rejected": -1.8327430486679077,
+      "loss": 1.2191,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6714308261871338,
+      "rewards/margins": 0.1613123118877411,
+      "rewards/rejected": -1.8327430486679077,
+      "semantic_entropy": 0.6718080639839172,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 11.416918885286389,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": 0.03169601410627365,
+      "logits/rejected": 0.17871348559856415,
+      "logps/chosen": -1.604529619216919,
+      "logps/rejected": -1.7868423461914062,
+      "loss": 1.1626,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.604529619216919,
+      "rewards/margins": 0.18231286108493805,
+      "rewards/rejected": -1.7868423461914062,
+      "semantic_entropy": 0.6853445768356323,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 8.025754305501584,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": 0.0022910747211426497,
+      "logits/rejected": 0.13192196190357208,
+      "logps/chosen": -1.6238453388214111,
+      "logps/rejected": -1.8105382919311523,
+      "loss": 1.1852,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.6238453388214111,
+      "rewards/margins": 0.18669314682483673,
+      "rewards/rejected": -1.8105382919311523,
+      "semantic_entropy": 0.6686668992042542,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 8.463069645500966,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": 0.03698580339550972,
+      "logits/rejected": 0.15946617722511292,
+      "logps/chosen": -1.6882175207138062,
+      "logps/rejected": -1.8159548044204712,
+      "loss": 1.1946,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6882175207138062,
+      "rewards/margins": 0.12773720920085907,
+      "rewards/rejected": -1.8159548044204712,
+      "semantic_entropy": 0.6628464460372925,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 10.676799942265538,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": -0.030872127041220665,
+      "logits/rejected": 0.14204931259155273,
+      "logps/chosen": -1.7597668170928955,
+      "logps/rejected": -1.822314977645874,
+      "loss": 1.2731,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.7597668170928955,
+      "rewards/margins": 0.06254833191633224,
+      "rewards/rejected": -1.822314977645874,
+      "semantic_entropy": 0.6572608947753906,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 8.336132259995036,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": 0.0017228275537490845,
+      "logits/rejected": 0.15485692024230957,
+      "logps/chosen": -1.5627415180206299,
+      "logps/rejected": -1.770357370376587,
+      "loss": 1.1831,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5627415180206299,
+      "rewards/margins": 0.20761580765247345,
+      "rewards/rejected": -1.770357370376587,
+      "semantic_entropy": 0.6934438943862915,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 8.326248295493343,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.2217852771282196,
+      "logits/rejected": -0.1151612251996994,
+      "logps/chosen": -1.7564977407455444,
+      "logps/rejected": -1.8627268075942993,
+      "loss": 1.1609,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.7564977407455444,
+      "rewards/margins": 0.10622922331094742,
+      "rewards/rejected": -1.8627268075942993,
+      "semantic_entropy": 0.6415423154830933,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 13.892072587579522,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": -0.06825049221515656,
+      "logits/rejected": 0.02035336196422577,
+      "logps/chosen": -1.780282974243164,
+      "logps/rejected": -1.8919916152954102,
+      "loss": 1.2118,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.780282974243164,
+      "rewards/margins": 0.11170871555805206,
+      "rewards/rejected": -1.8919916152954102,
+      "semantic_entropy": 0.6201997995376587,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 6.043187801813994,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": -0.08397931605577469,
+      "logits/rejected": 0.05286841467022896,
+      "logps/chosen": -1.6530301570892334,
+      "logps/rejected": -1.7906677722930908,
+      "loss": 1.1742,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.6530301570892334,
+      "rewards/margins": 0.13763758540153503,
+      "rewards/rejected": -1.7906677722930908,
+      "semantic_entropy": 0.659746527671814,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 8.586028996787945,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": -0.023158282041549683,
+      "logits/rejected": 0.07226009666919708,
+      "logps/chosen": -1.625187635421753,
+      "logps/rejected": -1.880324363708496,
+      "loss": 1.1784,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.625187635421753,
+      "rewards/margins": 0.25513654947280884,
+      "rewards/rejected": -1.880324363708496,
+      "semantic_entropy": 0.6813799142837524,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 19.29845663494889,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": -0.10296851396560669,
+      "logits/rejected": 0.0632767453789711,
+      "logps/chosen": -1.7459930181503296,
+      "logps/rejected": -1.8822393417358398,
+      "loss": 1.19,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.7459930181503296,
+      "rewards/margins": 0.13624632358551025,
+      "rewards/rejected": -1.8822393417358398,
+      "semantic_entropy": 0.635105311870575,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 7.1483216936330205,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": -0.08144383877515793,
+      "logits/rejected": 0.059574536979198456,
+      "logps/chosen": -1.7078588008880615,
+      "logps/rejected": -1.8417026996612549,
+      "loss": 1.2258,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.7078588008880615,
+      "rewards/margins": 0.13384383916854858,
+      "rewards/rejected": -1.8417026996612549,
+      "semantic_entropy": 0.6546602845191956,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 11.25048427915067,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": -0.13591568171977997,
+      "logits/rejected": 0.1586589217185974,
+      "logps/chosen": -1.6742719411849976,
+      "logps/rejected": -1.9009685516357422,
+      "loss": 1.1098,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6742719411849976,
+      "rewards/margins": 0.22669664025306702,
+      "rewards/rejected": -1.9009685516357422,
+      "semantic_entropy": 0.6484723687171936,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 19.244365719793826,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": -0.05710665509104729,
+      "logits/rejected": 0.002394887851551175,
+      "logps/chosen": -1.706974744796753,
+      "logps/rejected": -1.8144944906234741,
+      "loss": 1.1798,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.706974744796753,
+      "rewards/margins": 0.10751974582672119,
+      "rewards/rejected": -1.8144944906234741,
+      "semantic_entropy": 0.6426698565483093,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 11.090045319004785,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": -0.09708189219236374,
+      "logits/rejected": 0.06888498365879059,
+      "logps/chosen": -1.829058051109314,
+      "logps/rejected": -1.9642412662506104,
+      "loss": 1.18,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.829058051109314,
+      "rewards/margins": 0.1351831555366516,
+      "rewards/rejected": -1.9642412662506104,
+      "semantic_entropy": 0.6188846230506897,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 8.6001323792822,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": -0.02803843654692173,
+      "logits/rejected": 0.044818051159381866,
+      "logps/chosen": -1.9058525562286377,
+      "logps/rejected": -1.8785909414291382,
+      "loss": 1.2418,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.9058525562286377,
+      "rewards/margins": -0.02726154960691929,
+      "rewards/rejected": -1.8785909414291382,
+      "semantic_entropy": 0.5945314168930054,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 14.03321126952602,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.20716170966625214,
+      "logits/rejected": -0.11611819267272949,
+      "logps/chosen": -1.8215583562850952,
+      "logps/rejected": -1.9699954986572266,
+      "loss": 1.1671,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.8215583562850952,
+      "rewards/margins": 0.1484372913837433,
+      "rewards/rejected": -1.9699954986572266,
+      "semantic_entropy": 0.605375349521637,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 20.109440716151433,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": -0.01543907355517149,
+      "logits/rejected": 0.15439024567604065,
+      "logps/chosen": -1.8279422521591187,
+      "logps/rejected": -2.045393228530884,
+      "loss": 1.1385,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8279422521591187,
+      "rewards/margins": 0.217450812458992,
+      "rewards/rejected": -2.045393228530884,
+      "semantic_entropy": 0.6069425344467163,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 9.605340148780238,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": -0.06758487969636917,
+      "logits/rejected": 0.06253887712955475,
+      "logps/chosen": -1.7215917110443115,
+      "logps/rejected": -1.8040058612823486,
+      "loss": 1.1902,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7215917110443115,
+      "rewards/margins": 0.08241388946771622,
+      "rewards/rejected": -1.8040058612823486,
+      "semantic_entropy": 0.6455724835395813,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 13.039062337847668,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.13304288685321808,
+      "logits/rejected": -0.012511616572737694,
+      "logps/chosen": -1.773850679397583,
+      "logps/rejected": -2.1344165802001953,
+      "loss": 1.1068,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.773850679397583,
+      "rewards/margins": 0.360566109418869,
+      "rewards/rejected": -2.1344165802001953,
+      "semantic_entropy": 0.6139716506004333,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 12.94757740456836,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": -0.013867738656699657,
+      "logits/rejected": 0.14158634841442108,
+      "logps/chosen": -1.7584102153778076,
+      "logps/rejected": -2.1292402744293213,
+      "loss": 1.0564,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7584102153778076,
+      "rewards/margins": 0.3708299398422241,
+      "rewards/rejected": -2.1292402744293213,
+      "semantic_entropy": 0.6093857884407043,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 19.440072669225675,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.014184916391968727,
+      "logits/rejected": 0.12469978630542755,
+      "logps/chosen": -1.8560062646865845,
+      "logps/rejected": -1.969351053237915,
+      "loss": 1.1634,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8560062646865845,
+      "rewards/margins": 0.11334502696990967,
+      "rewards/rejected": -1.969351053237915,
+      "semantic_entropy": 0.5981619358062744,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 22.958436236488758,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": -0.039122484624385834,
+      "logits/rejected": 0.11318914592266083,
+      "logps/chosen": -2.0470094680786133,
+      "logps/rejected": -2.136540174484253,
+      "loss": 1.242,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.0470094680786133,
+      "rewards/margins": 0.08953092247247696,
+      "rewards/rejected": -2.136540174484253,
+      "semantic_entropy": 0.5616099834442139,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 14.744622568218817,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.06445063650608063,
+      "logits/rejected": 0.09797215461730957,
+      "logps/chosen": -1.89496648311615,
+      "logps/rejected": -2.1372017860412598,
+      "loss": 1.1548,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.89496648311615,
+      "rewards/margins": 0.24223539233207703,
+      "rewards/rejected": -2.1372017860412598,
+      "semantic_entropy": 0.5796412229537964,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 25.846182951058644,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": 0.024831080809235573,
+      "logits/rejected": 0.12422195822000504,
+      "logps/chosen": -1.8359193801879883,
+      "logps/rejected": -1.9778085947036743,
+      "loss": 1.187,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8359193801879883,
+      "rewards/margins": 0.14188924431800842,
+      "rewards/rejected": -1.9778085947036743,
+      "semantic_entropy": 0.6102578043937683,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 11.018506967938219,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": -0.07977452874183655,
+      "logits/rejected": 0.1439720243215561,
+      "logps/chosen": -1.8616647720336914,
+      "logps/rejected": -1.9908180236816406,
+      "loss": 1.1519,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8616647720336914,
+      "rewards/margins": 0.12915344536304474,
+      "rewards/rejected": -1.9908180236816406,
+      "semantic_entropy": 0.590460479259491,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 9.89563523605627,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": -0.02661723829805851,
+      "logits/rejected": 0.05380595847964287,
+      "logps/chosen": -1.8694766759872437,
+      "logps/rejected": -2.0911507606506348,
+      "loss": 1.135,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.8694766759872437,
+      "rewards/margins": 0.2216736376285553,
+      "rewards/rejected": -2.0911507606506348,
+      "semantic_entropy": 0.6067219972610474,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 22.840568359868527,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": 0.002448004437610507,
+      "logits/rejected": 0.08971460908651352,
+      "logps/chosen": -1.8331384658813477,
+      "logps/rejected": -1.915490746498108,
+      "loss": 1.1862,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.8331384658813477,
+      "rewards/margins": 0.08235229551792145,
+      "rewards/rejected": -1.915490746498108,
+      "semantic_entropy": 0.6140187978744507,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 13.141191279449925,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": -0.0027976795099675655,
+      "logits/rejected": 0.09904000163078308,
+      "logps/chosen": -1.811004638671875,
+      "logps/rejected": -1.944724678993225,
+      "loss": 1.1857,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.811004638671875,
+      "rewards/margins": 0.13371993601322174,
+      "rewards/rejected": -1.944724678993225,
+      "semantic_entropy": 0.6188271641731262,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 12.334363792290889,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": -0.037974268198013306,
+      "logits/rejected": 0.11785247176885605,
+      "logps/chosen": -1.844355583190918,
+      "logps/rejected": -2.170888662338257,
+      "loss": 1.0743,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.844355583190918,
+      "rewards/margins": 0.3265330493450165,
+      "rewards/rejected": -2.170888662338257,
+      "semantic_entropy": 0.5874147415161133,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 8.96572143368446,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": -0.0002848893345799297,
+      "logits/rejected": 0.08968429267406464,
+      "logps/chosen": -2.0417470932006836,
+      "logps/rejected": -2.3080222606658936,
+      "loss": 1.0521,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.0417470932006836,
+      "rewards/margins": 0.2662753760814667,
+      "rewards/rejected": -2.3080222606658936,
+      "semantic_entropy": 0.5384238958358765,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 7.584925396172419,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": 0.046282339841127396,
+      "logits/rejected": 0.12301240861415863,
+      "logps/chosen": -2.038160800933838,
+      "logps/rejected": -2.1695427894592285,
+      "loss": 1.1091,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.038160800933838,
+      "rewards/margins": 0.13138198852539062,
+      "rewards/rejected": -2.1695427894592285,
+      "semantic_entropy": 0.5332679748535156,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 12.846318550788451,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": 0.11053168773651123,
+      "logits/rejected": 0.2898208796977997,
+      "logps/chosen": -2.1209683418273926,
+      "logps/rejected": -2.3775408267974854,
+      "loss": 1.064,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.1209683418273926,
+      "rewards/margins": 0.2565723955631256,
+      "rewards/rejected": -2.3775408267974854,
+      "semantic_entropy": 0.5048622488975525,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 18.492330481149832,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": -0.008643044158816338,
+      "logits/rejected": 0.16993018984794617,
+      "logps/chosen": -2.0330655574798584,
+      "logps/rejected": -2.2131776809692383,
+      "loss": 1.1039,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.0330655574798584,
+      "rewards/margins": 0.18011203408241272,
+      "rewards/rejected": -2.2131776809692383,
+      "semantic_entropy": 0.5406839847564697,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 12.47190614587209,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": 0.10623683035373688,
+      "logits/rejected": 0.21453991532325745,
+      "logps/chosen": -2.154700517654419,
+      "logps/rejected": -2.3573577404022217,
+      "loss": 1.0568,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.154700517654419,
+      "rewards/margins": 0.2026570737361908,
+      "rewards/rejected": -2.3573577404022217,
+      "semantic_entropy": 0.5086141228675842,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": 0.3652660548686981,
+      "eval_logits/rejected": 0.4623660445213318,
+      "eval_logps/chosen": -2.1119725704193115,
+      "eval_logps/rejected": -2.395751476287842,
+      "eval_loss": 1.0641472339630127,
+      "eval_rewards/accuracies": 0.5497032403945923,
+      "eval_rewards/chosen": -2.1119725704193115,
+      "eval_rewards/margins": 0.283779114484787,
+      "eval_rewards/rejected": -2.395751476287842,
+      "eval_runtime": 35.3785,
+      "eval_samples_per_second": 38.017,
+      "eval_semantic_entropy": 0.5088942646980286,
+      "eval_steps_per_second": 9.526,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 11.488398494359684,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": 0.04418354481458664,
+      "logits/rejected": 0.13797423243522644,
+      "logps/chosen": -2.1950936317443848,
+      "logps/rejected": -2.4859559535980225,
+      "loss": 1.1138,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.1950936317443848,
+      "rewards/margins": 0.290862500667572,
+      "rewards/rejected": -2.4859559535980225,
+      "semantic_entropy": 0.49778300523757935,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 19.121226468685848,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": 0.060846053063869476,
+      "logits/rejected": 0.18392091989517212,
+      "logps/chosen": -2.0888547897338867,
+      "logps/rejected": -2.3829102516174316,
+      "loss": 1.048,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0888547897338867,
+      "rewards/margins": 0.2940555512905121,
+      "rewards/rejected": -2.3829102516174316,
+      "semantic_entropy": 0.5067925453186035,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 11.525744758497213,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": 0.06735793501138687,
+      "logits/rejected": 0.1239793449640274,
+      "logps/chosen": -2.148158550262451,
+      "logps/rejected": -2.3699960708618164,
+      "loss": 1.0534,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.148158550262451,
+      "rewards/margins": 0.22183747589588165,
+      "rewards/rejected": -2.3699960708618164,
+      "semantic_entropy": 0.5024824142456055,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 11.135439356622006,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": 0.01264535915106535,
+      "logits/rejected": 0.23167631030082703,
+      "logps/chosen": -2.0301260948181152,
+      "logps/rejected": -2.2477946281433105,
+      "loss": 1.0701,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -2.0301260948181152,
+      "rewards/margins": 0.21766862273216248,
+      "rewards/rejected": -2.2477946281433105,
+      "semantic_entropy": 0.523754358291626,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 18.104910657837852,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": -0.0024508326314389706,
+      "logits/rejected": 0.20202407240867615,
+      "logps/chosen": -2.1529297828674316,
+      "logps/rejected": -2.477417230606079,
+      "loss": 0.9906,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1529297828674316,
+      "rewards/margins": 0.3244874179363251,
+      "rewards/rejected": -2.477417230606079,
+      "semantic_entropy": 0.4880361557006836,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 13.566753095803973,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": -0.02302130125463009,
+      "logits/rejected": 0.18629023432731628,
+      "logps/chosen": -2.1584887504577637,
+      "logps/rejected": -2.6789536476135254,
+      "loss": 0.9794,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1584887504577637,
+      "rewards/margins": 0.5204650163650513,
+      "rewards/rejected": -2.6789536476135254,
+      "semantic_entropy": 0.48945674300193787,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 31.18205907167199,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": 0.07804682105779648,
+      "logits/rejected": 0.1684618890285492,
+      "logps/chosen": -2.217855215072632,
+      "logps/rejected": -2.43171763420105,
+      "loss": 1.0155,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.217855215072632,
+      "rewards/margins": 0.21386244893074036,
+      "rewards/rejected": -2.43171763420105,
+      "semantic_entropy": 0.47441864013671875,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 25.794826534493193,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": 0.07157039642333984,
+      "logits/rejected": 0.1782304346561432,
+      "logps/chosen": -2.22529673576355,
+      "logps/rejected": -2.5967438220977783,
+      "loss": 0.9762,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.22529673576355,
+      "rewards/margins": 0.37144696712493896,
+      "rewards/rejected": -2.5967438220977783,
+      "semantic_entropy": 0.46400684118270874,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 18.154539341479733,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": 0.060744207352399826,
+      "logits/rejected": 0.17653286457061768,
+      "logps/chosen": -2.3335304260253906,
+      "logps/rejected": -2.8038604259490967,
+      "loss": 0.9648,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.3335304260253906,
+      "rewards/margins": 0.470330148935318,
+      "rewards/rejected": -2.8038604259490967,
+      "semantic_entropy": 0.43205586075782776,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 26.405398664908027,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": 0.09718057513237,
+      "logits/rejected": 0.2404143363237381,
+      "logps/chosen": -2.415076732635498,
+      "logps/rejected": -2.7759289741516113,
+      "loss": 0.9307,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.415076732635498,
+      "rewards/margins": 0.36085230112075806,
+      "rewards/rejected": -2.7759289741516113,
+      "semantic_entropy": 0.40991273522377014,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 33.73453991404575,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": 0.11959276348352432,
+      "logits/rejected": 0.19597719609737396,
+      "logps/chosen": -2.354644536972046,
+      "logps/rejected": -2.8266985416412354,
+      "loss": 0.9176,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.354644536972046,
+      "rewards/margins": 0.47205406427383423,
+      "rewards/rejected": -2.8266985416412354,
+      "semantic_entropy": 0.41402873396873474,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 14.521645748953027,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": -0.011077843606472015,
+      "logits/rejected": 0.1186801940202713,
+      "logps/chosen": -2.5941014289855957,
+      "logps/rejected": -2.9222397804260254,
+      "loss": 0.9296,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.5941014289855957,
+      "rewards/margins": 0.3281381130218506,
+      "rewards/rejected": -2.9222397804260254,
+      "semantic_entropy": 0.36336058378219604,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 19.7157074234429,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": 0.23275712132453918,
+      "logits/rejected": 0.2626512050628662,
+      "logps/chosen": -2.7716641426086426,
+      "logps/rejected": -3.0999791622161865,
+      "loss": 0.9182,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.7716641426086426,
+      "rewards/margins": 0.3283149302005768,
+      "rewards/rejected": -3.0999791622161865,
+      "semantic_entropy": 0.3286034166812897,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 19.4218748229055,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": 0.28018778562545776,
+      "logits/rejected": 0.21975938975811005,
+      "logps/chosen": -2.8292651176452637,
+      "logps/rejected": -3.1870062351226807,
+      "loss": 0.9353,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.8292651176452637,
+      "rewards/margins": 0.3577413260936737,
+      "rewards/rejected": -3.1870062351226807,
+      "semantic_entropy": 0.3103516697883606,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 15.793830504621358,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": 0.07193533331155777,
+      "logits/rejected": 0.21929597854614258,
+      "logps/chosen": -2.867928981781006,
+      "logps/rejected": -3.550222396850586,
+      "loss": 0.8246,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.867928981781006,
+      "rewards/margins": 0.6822931170463562,
+      "rewards/rejected": -3.550222396850586,
+      "semantic_entropy": 0.27993419766426086,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 18.118923544979978,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": 0.14581528306007385,
+      "logits/rejected": 0.3329901695251465,
+      "logps/chosen": -2.974531650543213,
+      "logps/rejected": -3.3549561500549316,
+      "loss": 0.8492,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.974531650543213,
+      "rewards/margins": 0.38042473793029785,
+      "rewards/rejected": -3.3549561500549316,
+      "semantic_entropy": 0.27411144971847534,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 32.132431998023776,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": 0.1704220324754715,
+      "logits/rejected": 0.22892165184020996,
+      "logps/chosen": -3.4023804664611816,
+      "logps/rejected": -3.7786765098571777,
+      "loss": 0.8325,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -3.4023804664611816,
+      "rewards/margins": 0.3762962520122528,
+      "rewards/rejected": -3.7786765098571777,
+      "semantic_entropy": 0.21149472892284393,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 23.823234240000538,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": 0.23311415314674377,
+      "logits/rejected": 0.29134106636047363,
+      "logps/chosen": -3.7429115772247314,
+      "logps/rejected": -4.012482166290283,
+      "loss": 0.8668,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -3.7429115772247314,
+      "rewards/margins": 0.26957008242607117,
+      "rewards/rejected": -4.012482166290283,
+      "semantic_entropy": 0.15878012776374817,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 27.262752453822404,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": 0.20356643199920654,
+      "logits/rejected": 0.22003726661205292,
+      "logps/chosen": -4.065051078796387,
+      "logps/rejected": -4.214099884033203,
+      "loss": 0.8836,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -4.065051078796387,
+      "rewards/margins": 0.1490480601787567,
+      "rewards/rejected": -4.214099884033203,
+      "semantic_entropy": 0.12934216856956482,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 29.571651698286296,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": 0.2222503423690796,
+      "logits/rejected": 0.2971336543560028,
+      "logps/chosen": -4.027997970581055,
+      "logps/rejected": -4.403872489929199,
+      "loss": 0.7553,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.027997970581055,
+      "rewards/margins": 0.37587469816207886,
+      "rewards/rejected": -4.403872489929199,
+      "semantic_entropy": 0.12529395520687103,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 29.1990543011583,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": 0.20725543797016144,
+      "logits/rejected": 0.3282544016838074,
+      "logps/chosen": -4.144927024841309,
+      "logps/rejected": -4.577089309692383,
+      "loss": 0.7054,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.144927024841309,
+      "rewards/margins": 0.4321621358394623,
+      "rewards/rejected": -4.577089309692383,
+      "semantic_entropy": 0.11362969875335693,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 15.328459504936525,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": 0.39779630303382874,
+      "logits/rejected": 0.4326993525028229,
+      "logps/chosen": -4.371044158935547,
+      "logps/rejected": -4.910982608795166,
+      "loss": 0.7149,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.371044158935547,
+      "rewards/margins": 0.5399381518363953,
+      "rewards/rejected": -4.910982608795166,
+      "semantic_entropy": 0.1071554645895958,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 23.094525415963613,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": 0.4349740147590637,
+      "logits/rejected": 0.5181924700737,
+      "logps/chosen": -4.6029229164123535,
+      "logps/rejected": -5.176184177398682,
+      "loss": 0.7118,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.6029229164123535,
+      "rewards/margins": 0.5732611417770386,
+      "rewards/rejected": -5.176184177398682,
+      "semantic_entropy": 0.092677041888237,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 28.358579274181395,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": 0.424426794052124,
+      "logits/rejected": 0.5173272490501404,
+      "logps/chosen": -5.070749759674072,
+      "logps/rejected": -5.61461877822876,
+      "loss": 0.7685,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.070749759674072,
+      "rewards/margins": 0.5438690781593323,
+      "rewards/rejected": -5.61461877822876,
+      "semantic_entropy": 0.0577317476272583,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 34.92389863973247,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": 0.47861409187316895,
+      "logits/rejected": 0.5591767430305481,
+      "logps/chosen": -5.25409460067749,
+      "logps/rejected": -5.802521705627441,
+      "loss": 0.7252,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.25409460067749,
+      "rewards/margins": 0.5484274625778198,
+      "rewards/rejected": -5.802521705627441,
+      "semantic_entropy": 0.05434073135256767,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 26.007839131382827,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": 0.45313674211502075,
+      "logits/rejected": 0.5224326252937317,
+      "logps/chosen": -4.978353977203369,
+      "logps/rejected": -5.489474773406982,
+      "loss": 0.735,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.978353977203369,
+      "rewards/margins": 0.5111205577850342,
+      "rewards/rejected": -5.489474773406982,
+      "semantic_entropy": 0.06892217695713043,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 23.629473729452133,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": 0.3985074758529663,
+      "logits/rejected": 0.5935748815536499,
+      "logps/chosen": -5.140198707580566,
+      "logps/rejected": -5.65228271484375,
+      "loss": 0.684,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.140198707580566,
+      "rewards/margins": 0.5120837092399597,
+      "rewards/rejected": -5.65228271484375,
+      "semantic_entropy": 0.058453164994716644,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 26.158508312094792,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": 0.48377031087875366,
+      "logits/rejected": 0.54566490650177,
+      "logps/chosen": -5.303767204284668,
+      "logps/rejected": -5.69875955581665,
+      "loss": 0.7145,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.303767204284668,
+      "rewards/margins": 0.3949924409389496,
+      "rewards/rejected": -5.69875955581665,
+      "semantic_entropy": 0.04748838022351265,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 25.97069306696979,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": 0.49374207854270935,
+      "logits/rejected": 0.6388667225837708,
+      "logps/chosen": -5.392495155334473,
+      "logps/rejected": -5.959937572479248,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.392495155334473,
+      "rewards/margins": 0.5674418807029724,
+      "rewards/rejected": -5.959937572479248,
+      "semantic_entropy": 0.0481238067150116,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 21.729526440206346,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": 0.6221501231193542,
+      "logits/rejected": 0.6451407670974731,
+      "logps/chosen": -5.846024513244629,
+      "logps/rejected": -6.279688358306885,
+      "loss": 0.7015,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.846024513244629,
+      "rewards/margins": 0.433664470911026,
+      "rewards/rejected": -6.279688358306885,
+      "semantic_entropy": 0.03069642186164856,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 39.95600343652393,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": 0.6401647925376892,
+      "logits/rejected": 0.7274417877197266,
+      "logps/chosen": -6.000155448913574,
+      "logps/rejected": -6.236788749694824,
+      "loss": 0.7685,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -6.000155448913574,
+      "rewards/margins": 0.23663368821144104,
+      "rewards/rejected": -6.236788749694824,
+      "semantic_entropy": 0.027363503351807594,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 32.68813094112662,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": 0.6156709790229797,
+      "logits/rejected": 0.6307386159896851,
+      "logps/chosen": -5.798383712768555,
+      "logps/rejected": -6.192427158355713,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.798383712768555,
+      "rewards/margins": 0.39404380321502686,
+      "rewards/rejected": -6.192427158355713,
+      "semantic_entropy": 0.03181470185518265,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 20.248985489438827,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": 0.619035542011261,
+      "logits/rejected": 0.6660767793655396,
+      "logps/chosen": -5.881640434265137,
+      "logps/rejected": -6.391324043273926,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.881640434265137,
+      "rewards/margins": 0.5096836686134338,
+      "rewards/rejected": -6.391324043273926,
+      "semantic_entropy": 0.029121315106749535,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 22.90567646791892,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": 0.6241763234138489,
+      "logits/rejected": 0.7548576593399048,
+      "logps/chosen": -5.986024379730225,
+      "logps/rejected": -6.630023002624512,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -5.986024379730225,
+      "rewards/margins": 0.6439987421035767,
+      "rewards/rejected": -6.630023002624512,
+      "semantic_entropy": 0.027560269460082054,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 21.577140045601148,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": 0.7123579382896423,
+      "logits/rejected": 0.7100203633308411,
+      "logps/chosen": -6.294899940490723,
+      "logps/rejected": -6.742361545562744,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -6.294899940490723,
+      "rewards/margins": 0.44746190309524536,
+      "rewards/rejected": -6.742361545562744,
+      "semantic_entropy": 0.019608071073889732,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 23.54241703173219,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": 0.7639244794845581,
+      "logits/rejected": 0.84583979845047,
+      "logps/chosen": -6.595450401306152,
+      "logps/rejected": -6.977761745452881,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -6.595450401306152,
+      "rewards/margins": 0.3823109567165375,
+      "rewards/rejected": -6.977761745452881,
+      "semantic_entropy": 0.01550162024796009,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 22.761470158468605,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": 0.7423821091651917,
+      "logits/rejected": 0.7837462425231934,
+      "logps/chosen": -6.952185153961182,
+      "logps/rejected": -7.213634490966797,
+      "loss": 0.7412,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -6.952185153961182,
+      "rewards/margins": 0.26144999265670776,
+      "rewards/rejected": -7.213634490966797,
+      "semantic_entropy": 0.011904408223927021,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 38.74765536151773,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": 0.8192524909973145,
+      "logits/rejected": 0.8762401342391968,
+      "logps/chosen": -7.174726963043213,
+      "logps/rejected": -7.487733364105225,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.174726963043213,
+      "rewards/margins": 0.31300702691078186,
+      "rewards/rejected": -7.487733364105225,
+      "semantic_entropy": 0.009485157206654549,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 25.526849121164112,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": 0.8842913508415222,
+      "logits/rejected": 0.9352032542228699,
+      "logps/chosen": -7.037677764892578,
+      "logps/rejected": -7.461971282958984,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -7.037677764892578,
+      "rewards/margins": 0.4242933392524719,
+      "rewards/rejected": -7.461971282958984,
+      "semantic_entropy": 0.009824407286942005,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 19.15742707909043,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": 0.7710654139518738,
+      "logits/rejected": 0.8650022745132446,
+      "logps/chosen": -6.457311153411865,
+      "logps/rejected": -6.88073205947876,
+      "loss": 0.6387,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -6.457311153411865,
+      "rewards/margins": 0.4234209954738617,
+      "rewards/rejected": -6.88073205947876,
+      "semantic_entropy": 0.018056998029351234,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 24.84892358696139,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": 0.762574315071106,
+      "logits/rejected": 0.8002282977104187,
+      "logps/chosen": -6.4428815841674805,
+      "logps/rejected": -6.679665565490723,
+      "loss": 0.7565,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -6.4428815841674805,
+      "rewards/margins": 0.23678362369537354,
+      "rewards/rejected": -6.679665565490723,
+      "semantic_entropy": 0.018309544771909714,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 39.280150145870344,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": 0.8493047952651978,
+      "logits/rejected": 0.9356402158737183,
+      "logps/chosen": -6.415799617767334,
+      "logps/rejected": -6.9142632484436035,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -6.415799617767334,
+      "rewards/margins": 0.498462975025177,
+      "rewards/rejected": -6.9142632484436035,
+      "semantic_entropy": 0.016734320670366287,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 20.692184890704628,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": 0.9845502972602844,
+      "logits/rejected": 1.0035096406936646,
+      "logps/chosen": -7.00194787979126,
+      "logps/rejected": -7.516139030456543,
+      "loss": 0.5756,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.00194787979126,
+      "rewards/margins": 0.514191746711731,
+      "rewards/rejected": -7.516139030456543,
+      "semantic_entropy": 0.010593272745609283,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 21.374532051630606,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": 0.9575940370559692,
+      "logits/rejected": 0.9878286123275757,
+      "logps/chosen": -7.0054121017456055,
+      "logps/rejected": -7.618987083435059,
+      "loss": 0.5791,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -7.0054121017456055,
+      "rewards/margins": 0.6135755777359009,
+      "rewards/rejected": -7.618987083435059,
+      "semantic_entropy": 0.011255325749516487,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 22.89085035685026,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": 0.7985981106758118,
+      "logits/rejected": 0.8557677268981934,
+      "logps/chosen": -6.722765922546387,
+      "logps/rejected": -7.252674102783203,
+      "loss": 0.6498,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -6.722765922546387,
+      "rewards/margins": 0.5299077033996582,
+      "rewards/rejected": -7.252674102783203,
+      "semantic_entropy": 0.01515459269285202,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 29.3185640848479,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": 0.6307533383369446,
+      "logits/rejected": 0.7067631483078003,
+      "logps/chosen": -6.408628940582275,
+      "logps/rejected": -7.148467063903809,
+      "loss": 0.5587,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -6.408628940582275,
+      "rewards/margins": 0.7398383617401123,
+      "rewards/rejected": -7.148467063903809,
+      "semantic_entropy": 0.017853721976280212,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 19.14218879117827,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": 0.7474575638771057,
+      "logits/rejected": 0.8650363683700562,
+      "logps/chosen": -6.829617500305176,
+      "logps/rejected": -7.422484397888184,
+      "loss": 0.5928,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -6.829617500305176,
+      "rewards/margins": 0.5928672552108765,
+      "rewards/rejected": -7.422484397888184,
+      "semantic_entropy": 0.013212360441684723,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 22.863089839442907,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": 0.8633905649185181,
+      "logits/rejected": 0.9714609384536743,
+      "logps/chosen": -7.397378444671631,
+      "logps/rejected": -7.9788103103637695,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -7.397378444671631,
+      "rewards/margins": 0.58143150806427,
+      "rewards/rejected": -7.9788103103637695,
+      "semantic_entropy": 0.00779405701905489,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 20.806059209855995,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": 0.9922382235527039,
+      "logits/rejected": 0.9822915196418762,
+      "logps/chosen": -7.558897495269775,
+      "logps/rejected": -8.010220527648926,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -7.558897495269775,
+      "rewards/margins": 0.45132285356521606,
+      "rewards/rejected": -8.010220527648926,
+      "semantic_entropy": 0.006854074541479349,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 21.659161337091984,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": 0.9516774415969849,
+      "logits/rejected": 1.0095821619033813,
+      "logps/chosen": -7.3753461837768555,
+      "logps/rejected": -7.900918006896973,
+      "loss": 0.6241,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -7.3753461837768555,
+      "rewards/margins": 0.525571346282959,
+      "rewards/rejected": -7.900918006896973,
+      "semantic_entropy": 0.007624185644090176,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 22.502911720058954,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": 0.8318825960159302,
+      "logits/rejected": 0.8761769533157349,
+      "logps/chosen": -6.6802263259887695,
+      "logps/rejected": -7.3155035972595215,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -6.6802263259887695,
+      "rewards/margins": 0.6352773904800415,
+      "rewards/rejected": -7.3155035972595215,
+      "semantic_entropy": 0.013869042508304119,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 17.195620631605244,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": 1.0026311874389648,
+      "logits/rejected": 1.0498124361038208,
+      "logps/chosen": -7.466650485992432,
+      "logps/rejected": -7.763609886169434,
+      "loss": 0.7079,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -7.466650485992432,
+      "rewards/margins": 0.2969593405723572,
+      "rewards/rejected": -7.763609886169434,
+      "semantic_entropy": 0.007821312174201012,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 22.147655078594678,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": 1.0635933876037598,
+      "logits/rejected": 1.10765540599823,
+      "logps/chosen": -7.840873718261719,
+      "logps/rejected": -8.185582160949707,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -7.840873718261719,
+      "rewards/margins": 0.34470778703689575,
+      "rewards/rejected": -8.185582160949707,
+      "semantic_entropy": 0.004889071919023991,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 17.691264559714455,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": 0.9752556681632996,
+      "logits/rejected": 1.0362536907196045,
+      "logps/chosen": -7.573011875152588,
+      "logps/rejected": -8.063619613647461,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.573011875152588,
+      "rewards/margins": 0.49060964584350586,
+      "rewards/rejected": -8.063619613647461,
+      "semantic_entropy": 0.0059285834431648254,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 23.93164213540213,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": 0.9128883481025696,
+      "logits/rejected": 0.9371824264526367,
+      "logps/chosen": -7.125925540924072,
+      "logps/rejected": -7.614901542663574,
+      "loss": 0.6055,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.125925540924072,
+      "rewards/margins": 0.4889759123325348,
+      "rewards/rejected": -7.614901542663574,
+      "semantic_entropy": 0.010352469980716705,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 24.490777535596337,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": 0.7637569904327393,
+      "logits/rejected": 0.8398861885070801,
+      "logps/chosen": -6.525779724121094,
+      "logps/rejected": -7.049033164978027,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -6.525779724121094,
+      "rewards/margins": 0.523253321647644,
+      "rewards/rejected": -7.049033164978027,
+      "semantic_entropy": 0.016407480463385582,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 31.327915522900565,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": 0.8254741430282593,
+      "logits/rejected": 0.914653480052948,
+      "logps/chosen": -6.830941677093506,
+      "logps/rejected": -7.471418857574463,
+      "loss": 0.5805,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -6.830941677093506,
+      "rewards/margins": 0.6404756903648376,
+      "rewards/rejected": -7.471418857574463,
+      "semantic_entropy": 0.011883153580129147,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 23.110778399644605,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": 0.8560277819633484,
+      "logits/rejected": 0.8688713908195496,
+      "logps/chosen": -6.890917778015137,
+      "logps/rejected": -7.365884304046631,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -6.890917778015137,
+      "rewards/margins": 0.47496622800827026,
+      "rewards/rejected": -7.365884304046631,
+      "semantic_entropy": 0.012819747440516949,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 19.612328152266286,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": 0.8849146962165833,
+      "logits/rejected": 0.9137980341911316,
+      "logps/chosen": -6.988187313079834,
+      "logps/rejected": -7.5175461769104,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -6.988187313079834,
+      "rewards/margins": 0.5293585658073425,
+      "rewards/rejected": -7.5175461769104,
+      "semantic_entropy": 0.012281564064323902,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 23.564355013249614,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": 0.8927298784255981,
+      "logits/rejected": 0.9363479614257812,
+      "logps/chosen": -7.232657432556152,
+      "logps/rejected": -7.49481201171875,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -7.232657432556152,
+      "rewards/margins": 0.26215431094169617,
+      "rewards/rejected": -7.49481201171875,
+      "semantic_entropy": 0.008307929150760174,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 18.90350395744547,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": 0.9425722360610962,
+      "logits/rejected": 1.0250909328460693,
+      "logps/chosen": -7.007621765136719,
+      "logps/rejected": -7.541088104248047,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -7.007621765136719,
+      "rewards/margins": 0.5334662199020386,
+      "rewards/rejected": -7.541088104248047,
+      "semantic_entropy": 0.010339006781578064,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 18.262728070800026,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": 0.9385613203048706,
+      "logits/rejected": 0.9966949224472046,
+      "logps/chosen": -6.752771854400635,
+      "logps/rejected": -7.30672550201416,
+      "loss": 0.5649,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -6.752771854400635,
+      "rewards/margins": 0.5539542436599731,
+      "rewards/rejected": -7.30672550201416,
+      "semantic_entropy": 0.013281004503369331,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 24.92834130574777,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": 0.9952586889266968,
+      "logits/rejected": 1.075966477394104,
+      "logps/chosen": -7.34256649017334,
+      "logps/rejected": -7.798489570617676,
+      "loss": 0.6278,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -7.34256649017334,
+      "rewards/margins": 0.4559224247932434,
+      "rewards/rejected": -7.798489570617676,
+      "semantic_entropy": 0.0078593073412776,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 20.959919200029677,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": 1.1536259651184082,
+      "logits/rejected": 1.2452328205108643,
+      "logps/chosen": -7.662139892578125,
+      "logps/rejected": -8.316943168640137,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -7.662139892578125,
+      "rewards/margins": 0.6548035740852356,
+      "rewards/rejected": -8.316943168640137,
+      "semantic_entropy": 0.006051418371498585,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 18.91725816097431,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": 1.1448419094085693,
+      "logits/rejected": 1.2089478969573975,
+      "logps/chosen": -7.753819465637207,
+      "logps/rejected": -8.298693656921387,
+      "loss": 0.5894,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.753819465637207,
+      "rewards/margins": 0.5448734760284424,
+      "rewards/rejected": -8.298693656921387,
+      "semantic_entropy": 0.005721682216972113,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 18.255118166645527,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": 1.0719139575958252,
+      "logits/rejected": 1.1247889995574951,
+      "logps/chosen": -7.724333763122559,
+      "logps/rejected": -8.193605422973633,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -7.724333763122559,
+      "rewards/margins": 0.4692714810371399,
+      "rewards/rejected": -8.193605422973633,
+      "semantic_entropy": 0.006012583617120981,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 19.010431166863412,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": 0.9980684518814087,
+      "logits/rejected": 1.0625993013381958,
+      "logps/chosen": -7.431620121002197,
+      "logps/rejected": -7.8636369705200195,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.431620121002197,
+      "rewards/margins": 0.4320172667503357,
+      "rewards/rejected": -7.8636369705200195,
+      "semantic_entropy": 0.007210834417492151,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 17.163528409763188,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": 1.0620861053466797,
+      "logits/rejected": 1.1340640783309937,
+      "logps/chosen": -7.6426544189453125,
+      "logps/rejected": -8.188091278076172,
+      "loss": 0.5812,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.6426544189453125,
+      "rewards/margins": 0.5454373955726624,
+      "rewards/rejected": -8.188091278076172,
+      "semantic_entropy": 0.0056674242950975895,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 18.843526276706953,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": 1.1717276573181152,
+      "logits/rejected": 1.2118477821350098,
+      "logps/chosen": -7.970569610595703,
+      "logps/rejected": -8.400880813598633,
+      "loss": 0.642,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -7.970569610595703,
+      "rewards/margins": 0.4303114414215088,
+      "rewards/rejected": -8.400880813598633,
+      "semantic_entropy": 0.004340524785220623,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 15.596857102743344,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": 1.1037204265594482,
+      "logits/rejected": 1.1781091690063477,
+      "logps/chosen": -7.823260307312012,
+      "logps/rejected": -8.337902069091797,
+      "loss": 0.608,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.823260307312012,
+      "rewards/margins": 0.5146416425704956,
+      "rewards/rejected": -8.337902069091797,
+      "semantic_entropy": 0.0046985941007733345,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 20.19587215237011,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": 1.1348432302474976,
+      "logits/rejected": 1.202247977256775,
+      "logps/chosen": -7.805598258972168,
+      "logps/rejected": -8.287191390991211,
+      "loss": 0.6362,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.805598258972168,
+      "rewards/margins": 0.48159274458885193,
+      "rewards/rejected": -8.287191390991211,
+      "semantic_entropy": 0.005225029774010181,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 17.252665761409965,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": 1.0858346223831177,
+      "logits/rejected": 1.1307106018066406,
+      "logps/chosen": -7.828278541564941,
+      "logps/rejected": -8.277189254760742,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -7.828278541564941,
+      "rewards/margins": 0.4489101469516754,
+      "rewards/rejected": -8.277189254760742,
+      "semantic_entropy": 0.004872217774391174,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 21.43542876490759,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": 1.1436165571212769,
+      "logits/rejected": 1.188211441040039,
+      "logps/chosen": -7.910584449768066,
+      "logps/rejected": -8.395587921142578,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -7.910584449768066,
+      "rewards/margins": 0.485003799200058,
+      "rewards/rejected": -8.395587921142578,
+      "semantic_entropy": 0.0046251751482486725,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 23.040881502779477,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": 1.1512320041656494,
+      "logits/rejected": 1.208785057067871,
+      "logps/chosen": -7.786923408508301,
+      "logps/rejected": -8.389103889465332,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.786923408508301,
+      "rewards/margins": 0.602181077003479,
+      "rewards/rejected": -8.389103889465332,
+      "semantic_entropy": 0.004978600423783064,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 16.337262588395667,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": 1.0322002172470093,
+      "logits/rejected": 1.1025149822235107,
+      "logps/chosen": -7.447482109069824,
+      "logps/rejected": -7.890681266784668,
+      "loss": 0.643,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -7.447482109069824,
+      "rewards/margins": 0.44319963455200195,
+      "rewards/rejected": -7.890681266784668,
+      "semantic_entropy": 0.006798197515308857,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 25.597109253383646,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": 0.9592108726501465,
+      "logits/rejected": 1.0467469692230225,
+      "logps/chosen": -7.535346031188965,
+      "logps/rejected": -7.856226921081543,
+      "loss": 0.662,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -7.535346031188965,
+      "rewards/margins": 0.32088151574134827,
+      "rewards/rejected": -7.856226921081543,
+      "semantic_entropy": 0.006068686954677105,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 22.528922860199337,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": 1.0269381999969482,
+      "logits/rejected": 1.1069440841674805,
+      "logps/chosen": -7.437158107757568,
+      "logps/rejected": -7.976033687591553,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -7.437158107757568,
+      "rewards/margins": 0.5388745665550232,
+      "rewards/rejected": -7.976033687591553,
+      "semantic_entropy": 0.006848669145256281,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 16.060094916361866,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": 1.0504968166351318,
+      "logits/rejected": 1.092958688735962,
+      "logps/chosen": -7.546501159667969,
+      "logps/rejected": -8.020711898803711,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.546501159667969,
+      "rewards/margins": 0.47421059012413025,
+      "rewards/rejected": -8.020711898803711,
+      "semantic_entropy": 0.006295763887465,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 14.584406937817631,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": 1.0945534706115723,
+      "logits/rejected": 1.1743541955947876,
+      "logps/chosen": -7.755645751953125,
+      "logps/rejected": -8.337748527526855,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.755645751953125,
+      "rewards/margins": 0.5821027755737305,
+      "rewards/rejected": -8.337748527526855,
+      "semantic_entropy": 0.0051445746794342995,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 16.370810971422486,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": 1.155608057975769,
+      "logits/rejected": 1.1928364038467407,
+      "logps/chosen": -7.847119331359863,
+      "logps/rejected": -8.32005786895752,
+      "loss": 0.602,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.847119331359863,
+      "rewards/margins": 0.472938597202301,
+      "rewards/rejected": -8.32005786895752,
+      "semantic_entropy": 0.004602386616170406,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 1.3145469427108765,
+      "eval_logits/rejected": 1.3571854829788208,
+      "eval_logps/chosen": -7.826427459716797,
+      "eval_logps/rejected": -8.356344223022461,
+      "eval_loss": 0.5817172527313232,
+      "eval_rewards/accuracies": 0.6854599118232727,
+      "eval_rewards/chosen": -7.826427459716797,
+      "eval_rewards/margins": 0.5299174785614014,
+      "eval_rewards/rejected": -8.356344223022461,
+      "eval_runtime": 34.733,
+      "eval_samples_per_second": 38.724,
+      "eval_semantic_entropy": 0.004687093663960695,
+      "eval_steps_per_second": 9.703,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 15.488100482994879,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": 1.12029230594635,
+      "logits/rejected": 1.1895859241485596,
+      "logps/chosen": -7.682509422302246,
+      "logps/rejected": -8.260069847106934,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.682509422302246,
+      "rewards/margins": 0.5775591135025024,
+      "rewards/rejected": -8.260069847106934,
+      "semantic_entropy": 0.005233421456068754,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 19.041688391554842,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": 1.0938098430633545,
+      "logits/rejected": 1.1298701763153076,
+      "logps/chosen": -7.622184753417969,
+      "logps/rejected": -8.008241653442383,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.622184753417969,
+      "rewards/margins": 0.38605815172195435,
+      "rewards/rejected": -8.008241653442383,
+      "semantic_entropy": 0.005883772857487202,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 19.849753771876816,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": 1.1031490564346313,
+      "logits/rejected": 1.156066656112671,
+      "logps/chosen": -7.3606767654418945,
+      "logps/rejected": -8.045862197875977,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.3606767654418945,
+      "rewards/margins": 0.6851853132247925,
+      "rewards/rejected": -8.045862197875977,
+      "semantic_entropy": 0.006920933723449707,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 15.341597249276859,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": 1.11069655418396,
+      "logits/rejected": 1.1375608444213867,
+      "logps/chosen": -7.709697723388672,
+      "logps/rejected": -8.272204399108887,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.709697723388672,
+      "rewards/margins": 0.5625075101852417,
+      "rewards/rejected": -8.272204399108887,
+      "semantic_entropy": 0.005633797496557236,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 19.354754540601363,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": 1.1033474206924438,
+      "logits/rejected": 1.1650965213775635,
+      "logps/chosen": -7.760741233825684,
+      "logps/rejected": -8.35003662109375,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.760741233825684,
+      "rewards/margins": 0.589295506477356,
+      "rewards/rejected": -8.35003662109375,
+      "semantic_entropy": 0.005395522806793451,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 17.651147626277105,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": 1.1307498216629028,
+      "logits/rejected": 1.2124989032745361,
+      "logps/chosen": -7.690742492675781,
+      "logps/rejected": -8.37562370300293,
+      "loss": 0.5438,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -7.690742492675781,
+      "rewards/margins": 0.6848812699317932,
+      "rewards/rejected": -8.37562370300293,
+      "semantic_entropy": 0.005630246829241514,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 17.262529630359975,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": 1.1543856859207153,
+      "logits/rejected": 1.2043951749801636,
+      "logps/chosen": -7.757083892822266,
+      "logps/rejected": -8.37588882446289,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -7.757083892822266,
+      "rewards/margins": 0.6188048124313354,
+      "rewards/rejected": -8.37588882446289,
+      "semantic_entropy": 0.005794099997729063,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 19.947716996937064,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": 1.0710715055465698,
+      "logits/rejected": 1.1881413459777832,
+      "logps/chosen": -7.926266670227051,
+      "logps/rejected": -8.457072257995605,
+      "loss": 0.6236,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.926266670227051,
+      "rewards/margins": 0.5308045148849487,
+      "rewards/rejected": -8.457072257995605,
+      "semantic_entropy": 0.004605499561876059,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 14.090826290554961,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": 1.1270358562469482,
+      "logits/rejected": 1.1703280210494995,
+      "logps/chosen": -7.642250061035156,
+      "logps/rejected": -8.120524406433105,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -7.642250061035156,
+      "rewards/margins": 0.4782750606536865,
+      "rewards/rejected": -8.120524406433105,
+      "semantic_entropy": 0.006598387844860554,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 25.291036312648806,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": 1.0042312145233154,
+      "logits/rejected": 1.0598022937774658,
+      "logps/chosen": -7.3255109786987305,
+      "logps/rejected": -7.8547539710998535,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.3255109786987305,
+      "rewards/margins": 0.529242217540741,
+      "rewards/rejected": -7.8547539710998535,
+      "semantic_entropy": 0.0072741322219371796,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 34.872046862854624,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": 0.9909623265266418,
+      "logits/rejected": 1.082794427871704,
+      "logps/chosen": -7.108693599700928,
+      "logps/rejected": -7.7845916748046875,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.108693599700928,
+      "rewards/margins": 0.6758978366851807,
+      "rewards/rejected": -7.7845916748046875,
+      "semantic_entropy": 0.009369050152599812,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 15.23728209260186,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": 0.9029892683029175,
+      "logits/rejected": 1.01302170753479,
+      "logps/chosen": -7.19481897354126,
+      "logps/rejected": -7.872830867767334,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.19481897354126,
+      "rewards/margins": 0.6780112981796265,
+      "rewards/rejected": -7.872830867767334,
+      "semantic_entropy": 0.008518267422914505,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 31.765902270182213,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": 1.0632402896881104,
+      "logits/rejected": 1.1236531734466553,
+      "logps/chosen": -7.526625156402588,
+      "logps/rejected": -8.15988540649414,
+      "loss": 0.6016,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.526625156402588,
+      "rewards/margins": 0.6332606077194214,
+      "rewards/rejected": -8.15988540649414,
+      "semantic_entropy": 0.006810999475419521,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 15.467786734754782,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": 1.078277349472046,
+      "logits/rejected": 1.1878407001495361,
+      "logps/chosen": -7.801666259765625,
+      "logps/rejected": -8.494610786437988,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.801666259765625,
+      "rewards/margins": 0.6929444074630737,
+      "rewards/rejected": -8.494610786437988,
+      "semantic_entropy": 0.005062278360128403,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 21.25500680836216,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": 1.1355454921722412,
+      "logits/rejected": 1.1800553798675537,
+      "logps/chosen": -7.885350704193115,
+      "logps/rejected": -8.322117805480957,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -7.885350704193115,
+      "rewards/margins": 0.4367665648460388,
+      "rewards/rejected": -8.322117805480957,
+      "semantic_entropy": 0.004876808263361454,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 14.35949309533108,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": 1.10565185546875,
+      "logits/rejected": 1.2104301452636719,
+      "logps/chosen": -7.781987190246582,
+      "logps/rejected": -8.458442687988281,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.781987190246582,
+      "rewards/margins": 0.6764558553695679,
+      "rewards/rejected": -8.458442687988281,
+      "semantic_entropy": 0.0049747503362596035,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 17.47562370057817,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": 1.1020643711090088,
+      "logits/rejected": 1.1532132625579834,
+      "logps/chosen": -7.828910827636719,
+      "logps/rejected": -8.377799034118652,
+      "loss": 0.5876,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -7.828910827636719,
+      "rewards/margins": 0.5488887429237366,
+      "rewards/rejected": -8.377799034118652,
+      "semantic_entropy": 0.00481075793504715,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 19.950225695578762,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": 1.0718066692352295,
+      "logits/rejected": 1.1278977394104004,
+      "logps/chosen": -7.728150367736816,
+      "logps/rejected": -8.210725784301758,
+      "loss": 0.6004,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.728150367736816,
+      "rewards/margins": 0.4825774133205414,
+      "rewards/rejected": -8.210725784301758,
+      "semantic_entropy": 0.0054015410132706165,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 14.036809228128691,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": 1.1674606800079346,
+      "logits/rejected": 1.200852870941162,
+      "logps/chosen": -7.900933265686035,
+      "logps/rejected": -8.403078079223633,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.900933265686035,
+      "rewards/margins": 0.5021451711654663,
+      "rewards/rejected": -8.403078079223633,
+      "semantic_entropy": 0.004800423979759216,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 20.084115474024795,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": 1.147502064704895,
+      "logits/rejected": 1.189216136932373,
+      "logps/chosen": -7.914196968078613,
+      "logps/rejected": -8.43519401550293,
+      "loss": 0.6174,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.914196968078613,
+      "rewards/margins": 0.5209957957267761,
+      "rewards/rejected": -8.43519401550293,
+      "semantic_entropy": 0.004685095977038145,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 22.04947368841892,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": 1.151667594909668,
+      "logits/rejected": 1.2239385843276978,
+      "logps/chosen": -7.876222133636475,
+      "logps/rejected": -8.565794944763184,
+      "loss": 0.5515,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.876222133636475,
+      "rewards/margins": 0.689572811126709,
+      "rewards/rejected": -8.565794944763184,
+      "semantic_entropy": 0.005246061831712723,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 14.317132493668847,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": 1.12338387966156,
+      "logits/rejected": 1.176512360572815,
+      "logps/chosen": -7.805140018463135,
+      "logps/rejected": -8.295239448547363,
+      "loss": 0.6198,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -7.805140018463135,
+      "rewards/margins": 0.4900991916656494,
+      "rewards/rejected": -8.295239448547363,
+      "semantic_entropy": 0.005019624717533588,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 16.535640498018427,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": 1.119327187538147,
+      "logits/rejected": 1.193983793258667,
+      "logps/chosen": -7.739571571350098,
+      "logps/rejected": -8.318277359008789,
+      "loss": 0.5834,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -7.739571571350098,
+      "rewards/margins": 0.5787051916122437,
+      "rewards/rejected": -8.318277359008789,
+      "semantic_entropy": 0.005446913186460733,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 15.098846954280226,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": 1.0896193981170654,
+      "logits/rejected": 1.174743890762329,
+      "logps/chosen": -7.552855014801025,
+      "logps/rejected": -8.264971733093262,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.552855014801025,
+      "rewards/margins": 0.7121170163154602,
+      "rewards/rejected": -8.264971733093262,
+      "semantic_entropy": 0.006229312624782324,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 18.898677935750232,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": 1.0977445840835571,
+      "logits/rejected": 1.1614339351654053,
+      "logps/chosen": -7.836434841156006,
+      "logps/rejected": -8.285114288330078,
+      "loss": 0.6409,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -7.836434841156006,
+      "rewards/margins": 0.4486796259880066,
+      "rewards/rejected": -8.285114288330078,
+      "semantic_entropy": 0.004818376153707504,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 17.851782412691133,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": 1.1445541381835938,
+      "logits/rejected": 1.175467848777771,
+      "logps/chosen": -7.841614723205566,
+      "logps/rejected": -8.495734214782715,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -7.841614723205566,
+      "rewards/margins": 0.6541202664375305,
+      "rewards/rejected": -8.495734214782715,
+      "semantic_entropy": 0.004759738687425852,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 15.857679278196972,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": 1.2086812257766724,
+      "logits/rejected": 1.2444231510162354,
+      "logps/chosen": -8.026020050048828,
+      "logps/rejected": -8.60811710357666,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.026020050048828,
+      "rewards/margins": 0.5820974111557007,
+      "rewards/rejected": -8.60811710357666,
+      "semantic_entropy": 0.004328600596636534,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 17.3208213744782,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": 1.1870100498199463,
+      "logits/rejected": 1.2213929891586304,
+      "logps/chosen": -7.9112443923950195,
+      "logps/rejected": -8.487573623657227,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -7.9112443923950195,
+      "rewards/margins": 0.5763304829597473,
+      "rewards/rejected": -8.487573623657227,
+      "semantic_entropy": 0.004970698617398739,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 17.39164195785657,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": 1.1448256969451904,
+      "logits/rejected": 1.190725326538086,
+      "logps/chosen": -7.9466142654418945,
+      "logps/rejected": -8.467350959777832,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -7.9466142654418945,
+      "rewards/margins": 0.5207372903823853,
+      "rewards/rejected": -8.467350959777832,
+      "semantic_entropy": 0.005125958006829023,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 21.6320792199325,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": 1.124268651008606,
+      "logits/rejected": 1.2001936435699463,
+      "logps/chosen": -7.883705139160156,
+      "logps/rejected": -8.44405460357666,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.883705139160156,
+      "rewards/margins": 0.560349702835083,
+      "rewards/rejected": -8.44405460357666,
+      "semantic_entropy": 0.005186428315937519,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 19.107284309350362,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": 1.190467119216919,
+      "logits/rejected": 1.2714086771011353,
+      "logps/chosen": -8.265298843383789,
+      "logps/rejected": -8.905277252197266,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.265298843383789,
+      "rewards/margins": 0.6399778127670288,
+      "rewards/rejected": -8.905277252197266,
+      "semantic_entropy": 0.0038007772527635098,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 16.70759948273049,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": 1.2472827434539795,
+      "logits/rejected": 1.3009525537490845,
+      "logps/chosen": -8.464285850524902,
+      "logps/rejected": -9.103171348571777,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.464285850524902,
+      "rewards/margins": 0.6388850212097168,
+      "rewards/rejected": -9.103171348571777,
+      "semantic_entropy": 0.0034519669134169817,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 18.303592573413408,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": 1.289049506187439,
+      "logits/rejected": 1.3165349960327148,
+      "logps/chosen": -8.67881965637207,
+      "logps/rejected": -9.241315841674805,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -8.67881965637207,
+      "rewards/margins": 0.5624966621398926,
+      "rewards/rejected": -9.241315841674805,
+      "semantic_entropy": 0.0026764986105263233,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 17.380987944007902,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": 1.2492749691009521,
+      "logits/rejected": 1.2533103227615356,
+      "logps/chosen": -8.403242111206055,
+      "logps/rejected": -8.869034767150879,
+      "loss": 0.6167,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.403242111206055,
+      "rewards/margins": 0.46579235792160034,
+      "rewards/rejected": -8.869034767150879,
+      "semantic_entropy": 0.003303735051304102,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 13.561402687214796,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": 1.2426844835281372,
+      "logits/rejected": 1.3367140293121338,
+      "logps/chosen": -8.379229545593262,
+      "logps/rejected": -9.006486892700195,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.379229545593262,
+      "rewards/margins": 0.6272567510604858,
+      "rewards/rejected": -9.006486892700195,
+      "semantic_entropy": 0.003303577657788992,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 22.511595732827875,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": 1.1225228309631348,
+      "logits/rejected": 1.2004693746566772,
+      "logps/chosen": -8.20665168762207,
+      "logps/rejected": -8.796096801757812,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.20665168762207,
+      "rewards/margins": 0.5894451141357422,
+      "rewards/rejected": -8.796096801757812,
+      "semantic_entropy": 0.0036032279022037983,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 19.62747703459018,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": 1.073650598526001,
+      "logits/rejected": 1.1917431354522705,
+      "logps/chosen": -7.741274833679199,
+      "logps/rejected": -8.336652755737305,
+      "loss": 0.588,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -7.741274833679199,
+      "rewards/margins": 0.595378577709198,
+      "rewards/rejected": -8.336652755737305,
+      "semantic_entropy": 0.005601529963314533,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 14.636652059904227,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": 1.082037329673767,
+      "logits/rejected": 1.104501724243164,
+      "logps/chosen": -7.586031436920166,
+      "logps/rejected": -8.211699485778809,
+      "loss": 0.5825,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.586031436920166,
+      "rewards/margins": 0.6256687045097351,
+      "rewards/rejected": -8.211699485778809,
+      "semantic_entropy": 0.006120092235505581,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 16.715536030252302,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": 0.9997249841690063,
+      "logits/rejected": 1.043534278869629,
+      "logps/chosen": -7.416640281677246,
+      "logps/rejected": -7.989481449127197,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.416640281677246,
+      "rewards/margins": 0.5728405117988586,
+      "rewards/rejected": -7.989481449127197,
+      "semantic_entropy": 0.007317252457141876,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 14.443887302076575,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": 1.0232187509536743,
+      "logits/rejected": 1.1239763498306274,
+      "logps/chosen": -7.741781711578369,
+      "logps/rejected": -8.45250129699707,
+      "loss": 0.5298,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.741781711578369,
+      "rewards/margins": 0.7107199430465698,
+      "rewards/rejected": -8.45250129699707,
+      "semantic_entropy": 0.0055780112743377686,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 26.078733504253485,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": 1.062495231628418,
+      "logits/rejected": 1.157784342765808,
+      "logps/chosen": -7.902398586273193,
+      "logps/rejected": -8.475024223327637,
+      "loss": 0.589,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -7.902398586273193,
+      "rewards/margins": 0.5726253390312195,
+      "rewards/rejected": -8.475024223327637,
+      "semantic_entropy": 0.005477880127727985,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 20.38732219330966,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": 1.0688027143478394,
+      "logits/rejected": 1.1802704334259033,
+      "logps/chosen": -7.7444281578063965,
+      "logps/rejected": -8.503796577453613,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -7.7444281578063965,
+      "rewards/margins": 0.7593673467636108,
+      "rewards/rejected": -8.503796577453613,
+      "semantic_entropy": 0.006529929582029581,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 14.451384024062147,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": 1.1112804412841797,
+      "logits/rejected": 1.2086459398269653,
+      "logps/chosen": -7.976552486419678,
+      "logps/rejected": -8.61184310913086,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.976552486419678,
+      "rewards/margins": 0.635290265083313,
+      "rewards/rejected": -8.61184310913086,
+      "semantic_entropy": 0.004832428880035877,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 20.092163063025225,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": 1.1675083637237549,
+      "logits/rejected": 1.2536954879760742,
+      "logps/chosen": -7.943145751953125,
+      "logps/rejected": -8.56212043762207,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.943145751953125,
+      "rewards/margins": 0.6189749836921692,
+      "rewards/rejected": -8.56212043762207,
+      "semantic_entropy": 0.004479782655835152,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 18.543999868787445,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": 1.1390047073364258,
+      "logits/rejected": 1.2212417125701904,
+      "logps/chosen": -8.038432121276855,
+      "logps/rejected": -8.853678703308105,
+      "loss": 0.518,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.038432121276855,
+      "rewards/margins": 0.8152469396591187,
+      "rewards/rejected": -8.853678703308105,
+      "semantic_entropy": 0.004491516854614019,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 28.622405445212987,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": 1.1823326349258423,
+      "logits/rejected": 1.2506489753723145,
+      "logps/chosen": -8.058849334716797,
+      "logps/rejected": -8.665971755981445,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.058849334716797,
+      "rewards/margins": 0.6071220636367798,
+      "rewards/rejected": -8.665971755981445,
+      "semantic_entropy": 0.004828805569559336,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 16.088810777507515,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": 1.2168138027191162,
+      "logits/rejected": 1.2887437343597412,
+      "logps/chosen": -8.412802696228027,
+      "logps/rejected": -8.959291458129883,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.412802696228027,
+      "rewards/margins": 0.5464898347854614,
+      "rewards/rejected": -8.959291458129883,
+      "semantic_entropy": 0.0030591513495892286,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 15.42436292202903,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": 1.2541420459747314,
+      "logits/rejected": 1.3553508520126343,
+      "logps/chosen": -8.259259223937988,
+      "logps/rejected": -9.156826972961426,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.259259223937988,
+      "rewards/margins": 0.897567629814148,
+      "rewards/rejected": -9.156826972961426,
+      "semantic_entropy": 0.0040305061265826225,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 19.66169537700732,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": 1.2195287942886353,
+      "logits/rejected": 1.3087810277938843,
+      "logps/chosen": -8.59006118774414,
+      "logps/rejected": -9.23585033416748,
+      "loss": 0.5818,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.59006118774414,
+      "rewards/margins": 0.6457890272140503,
+      "rewards/rejected": -9.23585033416748,
+      "semantic_entropy": 0.002743345918133855,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 16.398237261756613,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": 1.1882917881011963,
+      "logits/rejected": 1.255778193473816,
+      "logps/chosen": -8.47636604309082,
+      "logps/rejected": -9.217564582824707,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.47636604309082,
+      "rewards/margins": 0.7411981225013733,
+      "rewards/rejected": -9.217564582824707,
+      "semantic_entropy": 0.0032881791703402996,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 19.49459061138459,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": 1.251731514930725,
+      "logits/rejected": 1.281177282333374,
+      "logps/chosen": -8.127111434936523,
+      "logps/rejected": -8.819128036499023,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.127111434936523,
+      "rewards/margins": 0.6920166611671448,
+      "rewards/rejected": -8.819128036499023,
+      "semantic_entropy": 0.003929547034204006,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 28.988323591485834,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": 1.2298028469085693,
+      "logits/rejected": 1.3155467510223389,
+      "logps/chosen": -8.147259712219238,
+      "logps/rejected": -8.793086051940918,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.147259712219238,
+      "rewards/margins": 0.6458262205123901,
+      "rewards/rejected": -8.793086051940918,
+      "semantic_entropy": 0.004039463587105274,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 17.475909423282587,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": 1.2026350498199463,
+      "logits/rejected": 1.2812477350234985,
+      "logps/chosen": -7.840292453765869,
+      "logps/rejected": -8.540803909301758,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.840292453765869,
+      "rewards/margins": 0.7005102038383484,
+      "rewards/rejected": -8.540803909301758,
+      "semantic_entropy": 0.0057488782331347466,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 15.953333150702546,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": 1.0297119617462158,
+      "logits/rejected": 1.1852972507476807,
+      "logps/chosen": -7.457418918609619,
+      "logps/rejected": -8.108861923217773,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -7.457418918609619,
+      "rewards/margins": 0.6514430046081543,
+      "rewards/rejected": -8.108861923217773,
+      "semantic_entropy": 0.0070342435501515865,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 12.907044345311284,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": 1.0394175052642822,
+      "logits/rejected": 1.1214516162872314,
+      "logps/chosen": -7.15374755859375,
+      "logps/rejected": -7.91888952255249,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.15374755859375,
+      "rewards/margins": 0.7651426196098328,
+      "rewards/rejected": -7.91888952255249,
+      "semantic_entropy": 0.011548938229680061,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 18.842353485054144,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": 1.0387752056121826,
+      "logits/rejected": 1.116888165473938,
+      "logps/chosen": -7.357861518859863,
+      "logps/rejected": -8.102581977844238,
+      "loss": 0.5715,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -7.357861518859863,
+      "rewards/margins": 0.7447201013565063,
+      "rewards/rejected": -8.102581977844238,
+      "semantic_entropy": 0.00840686447918415,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 17.517444586849834,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": 1.161381721496582,
+      "logits/rejected": 1.2572661638259888,
+      "logps/chosen": -7.585520267486572,
+      "logps/rejected": -8.227933883666992,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -7.585520267486572,
+      "rewards/margins": 0.6424126625061035,
+      "rewards/rejected": -8.227933883666992,
+      "semantic_entropy": 0.006788079626858234,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 17.69642023741119,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": 1.1936240196228027,
+      "logits/rejected": 1.2674013376235962,
+      "logps/chosen": -7.802135467529297,
+      "logps/rejected": -8.394062995910645,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.802135467529297,
+      "rewards/margins": 0.591926634311676,
+      "rewards/rejected": -8.394062995910645,
+      "semantic_entropy": 0.005987273063510656,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 16.735105390195496,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": 1.1332563161849976,
+      "logits/rejected": 1.2431234121322632,
+      "logps/chosen": -8.090959548950195,
+      "logps/rejected": -8.726110458374023,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.090959548950195,
+      "rewards/margins": 0.635151207447052,
+      "rewards/rejected": -8.726110458374023,
+      "semantic_entropy": 0.004236576613038778,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 18.405215104868617,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": 1.2462310791015625,
+      "logits/rejected": 1.3544846773147583,
+      "logps/chosen": -8.0702543258667,
+      "logps/rejected": -8.80981159210205,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.0702543258667,
+      "rewards/margins": 0.739556610584259,
+      "rewards/rejected": -8.80981159210205,
+      "semantic_entropy": 0.00429269764572382,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 18.411374709754437,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": 1.2096521854400635,
+      "logits/rejected": 1.2681427001953125,
+      "logps/chosen": -8.359540939331055,
+      "logps/rejected": -8.925297737121582,
+      "loss": 0.6021,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.359540939331055,
+      "rewards/margins": 0.565757155418396,
+      "rewards/rejected": -8.925297737121582,
+      "semantic_entropy": 0.003185038920491934,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 25.141970156016203,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": 1.3438087701797485,
+      "logits/rejected": 1.447033166885376,
+      "logps/chosen": -8.527837753295898,
+      "logps/rejected": -9.11477279663086,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.527837753295898,
+      "rewards/margins": 0.586935818195343,
+      "rewards/rejected": -9.11477279663086,
+      "semantic_entropy": 0.0028588466811925173,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 24.793957263345593,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": 1.3609850406646729,
+      "logits/rejected": 1.45890212059021,
+      "logps/chosen": -8.6663236618042,
+      "logps/rejected": -9.230647087097168,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.6663236618042,
+      "rewards/margins": 0.5643231868743896,
+      "rewards/rejected": -9.230647087097168,
+      "semantic_entropy": 0.002592557342723012,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 14.310044602410244,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": 1.3036677837371826,
+      "logits/rejected": 1.3909859657287598,
+      "logps/chosen": -8.649286270141602,
+      "logps/rejected": -9.332389831542969,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.649286270141602,
+      "rewards/margins": 0.6831027865409851,
+      "rewards/rejected": -9.332389831542969,
+      "semantic_entropy": 0.0025266888551414013,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 16.32270992453552,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": 1.2962621450424194,
+      "logits/rejected": 1.3894531726837158,
+      "logps/chosen": -8.858843803405762,
+      "logps/rejected": -9.499662399291992,
+      "loss": 0.5702,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.858843803405762,
+      "rewards/margins": 0.6408184766769409,
+      "rewards/rejected": -9.499662399291992,
+      "semantic_entropy": 0.002079838188365102,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 15.32965389360088,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": 1.391282320022583,
+      "logits/rejected": 1.4434236288070679,
+      "logps/chosen": -8.68392562866211,
+      "logps/rejected": -9.259500503540039,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.68392562866211,
+      "rewards/margins": 0.5755751132965088,
+      "rewards/rejected": -9.259500503540039,
+      "semantic_entropy": 0.0023343083448708057,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 18.672655843624852,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": 1.3818328380584717,
+      "logits/rejected": 1.4901525974273682,
+      "logps/chosen": -8.900357246398926,
+      "logps/rejected": -9.534480094909668,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.900357246398926,
+      "rewards/margins": 0.6341238617897034,
+      "rewards/rejected": -9.534480094909668,
+      "semantic_entropy": 0.0019445348298177123,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 16.55834003981296,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": 1.366276502609253,
+      "logits/rejected": 1.3809794187545776,
+      "logps/chosen": -8.882219314575195,
+      "logps/rejected": -9.380866050720215,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.882219314575195,
+      "rewards/margins": 0.4986463189125061,
+      "rewards/rejected": -9.380866050720215,
+      "semantic_entropy": 0.0019147871062159538,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 15.786299245286246,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": 1.3280194997787476,
+      "logits/rejected": 1.4059630632400513,
+      "logps/chosen": -8.86941909790039,
+      "logps/rejected": -9.583097457885742,
+      "loss": 0.5579,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.86941909790039,
+      "rewards/margins": 0.7136775851249695,
+      "rewards/rejected": -9.583097457885742,
+      "semantic_entropy": 0.0020566589664667845,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 16.665388307474053,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": 1.371848464012146,
+      "logits/rejected": 1.4409339427947998,
+      "logps/chosen": -9.003472328186035,
+      "logps/rejected": -9.680521011352539,
+      "loss": 0.5652,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.003472328186035,
+      "rewards/margins": 0.677047848701477,
+      "rewards/rejected": -9.680521011352539,
+      "semantic_entropy": 0.001840126933529973,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 11.798573099039368,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": 1.4037483930587769,
+      "logits/rejected": 1.4487072229385376,
+      "logps/chosen": -8.949201583862305,
+      "logps/rejected": -9.47164249420166,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -8.949201583862305,
+      "rewards/margins": 0.5224403142929077,
+      "rewards/rejected": -9.47164249420166,
+      "semantic_entropy": 0.002038115868344903,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 14.383197519556905,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": 1.3852488994598389,
+      "logits/rejected": 1.4487981796264648,
+      "logps/chosen": -8.819432258605957,
+      "logps/rejected": -9.531989097595215,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.819432258605957,
+      "rewards/margins": 0.712556004524231,
+      "rewards/rejected": -9.531989097595215,
+      "semantic_entropy": 0.0024776014033704996,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 16.61890330150234,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": 1.3563627004623413,
+      "logits/rejected": 1.4070327281951904,
+      "logps/chosen": -8.92655086517334,
+      "logps/rejected": -9.366896629333496,
+      "loss": 0.6378,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.92655086517334,
+      "rewards/margins": 0.4403463900089264,
+      "rewards/rejected": -9.366896629333496,
+      "semantic_entropy": 0.0022483307402580976,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 17.870129760324115,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": 1.3795759677886963,
+      "logits/rejected": 1.456594705581665,
+      "logps/chosen": -9.06200122833252,
+      "logps/rejected": -9.604608535766602,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -9.06200122833252,
+      "rewards/margins": 0.5426072478294373,
+      "rewards/rejected": -9.604608535766602,
+      "semantic_entropy": 0.001933540217578411,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 14.210274423375122,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": 1.4009263515472412,
+      "logits/rejected": 1.5007972717285156,
+      "logps/chosen": -9.014759063720703,
+      "logps/rejected": -9.633633613586426,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.014759063720703,
+      "rewards/margins": 0.6188760995864868,
+      "rewards/rejected": -9.633633613586426,
+      "semantic_entropy": 0.0023038857616484165,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 15.235722121385995,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": 1.3878511190414429,
+      "logits/rejected": 1.4722470045089722,
+      "logps/chosen": -9.091385841369629,
+      "logps/rejected": -9.721710205078125,
+      "loss": 0.577,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.091385841369629,
+      "rewards/margins": 0.6303255558013916,
+      "rewards/rejected": -9.721710205078125,
+      "semantic_entropy": 0.0017910923343151808,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 13.956920677054695,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": 1.4306633472442627,
+      "logits/rejected": 1.496328592300415,
+      "logps/chosen": -8.981457710266113,
+      "logps/rejected": -9.808283805847168,
+      "loss": 0.524,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.981457710266113,
+      "rewards/margins": 0.8268264532089233,
+      "rewards/rejected": -9.808283805847168,
+      "semantic_entropy": 0.005975677166134119,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 21.266056286731576,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": 1.4279210567474365,
+      "logits/rejected": 1.4926308393478394,
+      "logps/chosen": -9.173555374145508,
+      "logps/rejected": -9.846899032592773,
+      "loss": 0.5566,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.173555374145508,
+      "rewards/margins": 0.6733446717262268,
+      "rewards/rejected": -9.846899032592773,
+      "semantic_entropy": 0.0019384182523936033,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 14.882233309978883,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": 1.4547145366668701,
+      "logits/rejected": 1.5194149017333984,
+      "logps/chosen": -8.938140869140625,
+      "logps/rejected": -9.677942276000977,
+      "loss": 0.5114,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.938140869140625,
+      "rewards/margins": 0.7398021221160889,
+      "rewards/rejected": -9.677942276000977,
+      "semantic_entropy": 0.001993780490010977,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 19.38501867261891,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": 1.4046881198883057,
+      "logits/rejected": 1.4453130960464478,
+      "logps/chosen": -9.048368453979492,
+      "logps/rejected": -9.566811561584473,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -9.048368453979492,
+      "rewards/margins": 0.5184433460235596,
+      "rewards/rejected": -9.566811561584473,
+      "semantic_entropy": 0.002064045052975416,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": 1.5818452835083008,
+      "eval_logits/rejected": 1.6397122144699097,
+      "eval_logps/chosen": -8.896148681640625,
+      "eval_logps/rejected": -9.607196807861328,
+      "eval_loss": 0.5509479641914368,
+      "eval_rewards/accuracies": 0.6995549201965332,
+      "eval_rewards/chosen": -8.896148681640625,
+      "eval_rewards/margins": 0.711047887802124,
+      "eval_rewards/rejected": -9.607196807861328,
+      "eval_runtime": 34.6222,
+      "eval_samples_per_second": 38.848,
+      "eval_semantic_entropy": 0.002138254465535283,
+      "eval_steps_per_second": 9.734,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 15.671260914333047,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": 1.3665152788162231,
+      "logits/rejected": 1.458276629447937,
+      "logps/chosen": -8.895729064941406,
+      "logps/rejected": -9.644877433776855,
+      "loss": 0.5233,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.895729064941406,
+      "rewards/margins": 0.7491487264633179,
+      "rewards/rejected": -9.644877433776855,
+      "semantic_entropy": 0.0021528464276343584,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 15.438986952061061,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": 1.36763596534729,
+      "logits/rejected": 1.4583923816680908,
+      "logps/chosen": -8.857190132141113,
+      "logps/rejected": -9.477605819702148,
+      "loss": 0.57,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.857190132141113,
+      "rewards/margins": 0.6204143762588501,
+      "rewards/rejected": -9.477605819702148,
+      "semantic_entropy": 0.002285307738929987,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 17.58387785479654,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": 1.2972025871276855,
+      "logits/rejected": 1.373787522315979,
+      "logps/chosen": -8.858536720275879,
+      "logps/rejected": -9.369037628173828,
+      "loss": 0.6006,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.858536720275879,
+      "rewards/margins": 0.5105008482933044,
+      "rewards/rejected": -9.369037628173828,
+      "semantic_entropy": 0.002081833779811859,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 14.477235698929201,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": 1.3965874910354614,
+      "logits/rejected": 1.4628775119781494,
+      "logps/chosen": -8.528948783874512,
+      "logps/rejected": -9.17314624786377,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.528948783874512,
+      "rewards/margins": 0.6441971659660339,
+      "rewards/rejected": -9.17314624786377,
+      "semantic_entropy": 0.003048532409593463,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 18.55826365115439,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": 1.2824848890304565,
+      "logits/rejected": 1.3806899785995483,
+      "logps/chosen": -8.71102523803711,
+      "logps/rejected": -9.30419635772705,
+      "loss": 0.5773,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.71102523803711,
+      "rewards/margins": 0.593170702457428,
+      "rewards/rejected": -9.30419635772705,
+      "semantic_entropy": 0.002504490315914154,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 16.878701335836936,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": 1.3385461568832397,
+      "logits/rejected": 1.392857313156128,
+      "logps/chosen": -8.838831901550293,
+      "logps/rejected": -9.329095840454102,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -8.838831901550293,
+      "rewards/margins": 0.4902641177177429,
+      "rewards/rejected": -9.329095840454102,
+      "semantic_entropy": 0.0019668086897581816,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 14.431715715470096,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": 1.375388503074646,
+      "logits/rejected": 1.455187439918518,
+      "logps/chosen": -8.989799499511719,
+      "logps/rejected": -9.553720474243164,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.989799499511719,
+      "rewards/margins": 0.5639193654060364,
+      "rewards/rejected": -9.553720474243164,
+      "semantic_entropy": 0.0018786315340548754,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 12.13457848809534,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": 1.3388803005218506,
+      "logits/rejected": 1.3925445079803467,
+      "logps/chosen": -8.972952842712402,
+      "logps/rejected": -9.530712127685547,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -8.972952842712402,
+      "rewards/margins": 0.5577596426010132,
+      "rewards/rejected": -9.530712127685547,
+      "semantic_entropy": 0.0019155361223965883,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 17.097276214992615,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": 1.339806318283081,
+      "logits/rejected": 1.417843222618103,
+      "logps/chosen": -8.790437698364258,
+      "logps/rejected": -9.454071044921875,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.790437698364258,
+      "rewards/margins": 0.6636320352554321,
+      "rewards/rejected": -9.454071044921875,
+      "semantic_entropy": 0.00213107792660594,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 22.841031996877746,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": 1.3600574731826782,
+      "logits/rejected": 1.4646940231323242,
+      "logps/chosen": -8.934052467346191,
+      "logps/rejected": -9.37693977355957,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -8.934052467346191,
+      "rewards/margins": 0.44288843870162964,
+      "rewards/rejected": -9.37693977355957,
+      "semantic_entropy": 0.001983584137633443,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 13.335832756253948,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": 1.3720840215682983,
+      "logits/rejected": 1.4387794733047485,
+      "logps/chosen": -8.70246410369873,
+      "logps/rejected": -9.370990753173828,
+      "loss": 0.5378,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.70246410369873,
+      "rewards/margins": 0.6685260534286499,
+      "rewards/rejected": -9.370990753173828,
+      "semantic_entropy": 0.002819368615746498,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 12.94253965043617,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": 1.3387447595596313,
+      "logits/rejected": 1.4097579717636108,
+      "logps/chosen": -8.577402114868164,
+      "logps/rejected": -9.29970932006836,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.577402114868164,
+      "rewards/margins": 0.7223077416419983,
+      "rewards/rejected": -9.29970932006836,
+      "semantic_entropy": 0.0025379578582942486,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 18.170333357141804,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": 1.3639682531356812,
+      "logits/rejected": 1.4590171575546265,
+      "logps/chosen": -8.407323837280273,
+      "logps/rejected": -8.99671459197998,
+      "loss": 0.5686,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.407323837280273,
+      "rewards/margins": 0.5893905162811279,
+      "rewards/rejected": -8.99671459197998,
+      "semantic_entropy": 0.0029364742804318666,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 19.152097593888094,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": 1.3170546293258667,
+      "logits/rejected": 1.3590672016143799,
+      "logps/chosen": -8.186147689819336,
+      "logps/rejected": -8.675069808959961,
+      "loss": 0.6161,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.186147689819336,
+      "rewards/margins": 0.48892301321029663,
+      "rewards/rejected": -8.675069808959961,
+      "semantic_entropy": 0.004085545893758535,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 14.384401894638263,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": 1.31429922580719,
+      "logits/rejected": 1.3738658428192139,
+      "logps/chosen": -8.341313362121582,
+      "logps/rejected": -8.903071403503418,
+      "loss": 0.6208,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -8.341313362121582,
+      "rewards/margins": 0.5617581605911255,
+      "rewards/rejected": -8.903071403503418,
+      "semantic_entropy": 0.003315623151138425,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 22.769947902563544,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": 1.3556791543960571,
+      "logits/rejected": 1.4623668193817139,
+      "logps/chosen": -8.42508316040039,
+      "logps/rejected": -9.105088233947754,
+      "loss": 0.5526,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.42508316040039,
+      "rewards/margins": 0.6800051927566528,
+      "rewards/rejected": -9.105088233947754,
+      "semantic_entropy": 0.002977838972583413,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 17.425245769816975,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": 1.376682996749878,
+      "logits/rejected": 1.4646713733673096,
+      "logps/chosen": -8.790547370910645,
+      "logps/rejected": -9.228643417358398,
+      "loss": 0.6164,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.790547370910645,
+      "rewards/margins": 0.4380955100059509,
+      "rewards/rejected": -9.228643417358398,
+      "semantic_entropy": 0.002260428387671709,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 15.048061234204843,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": 1.3563090562820435,
+      "logits/rejected": 1.4552792310714722,
+      "logps/chosen": -8.72165298461914,
+      "logps/rejected": -9.205158233642578,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.72165298461914,
+      "rewards/margins": 0.4835067689418793,
+      "rewards/rejected": -9.205158233642578,
+      "semantic_entropy": 0.002371445996686816,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 13.963605204974463,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": 1.3829644918441772,
+      "logits/rejected": 1.3941246271133423,
+      "logps/chosen": -8.58009147644043,
+      "logps/rejected": -9.320999145507812,
+      "loss": 0.5252,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.58009147644043,
+      "rewards/margins": 0.7409077882766724,
+      "rewards/rejected": -9.320999145507812,
+      "semantic_entropy": 0.002713502384722233,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 22.41040473785911,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": 1.3895207643508911,
+      "logits/rejected": 1.5379985570907593,
+      "logps/chosen": -8.698478698730469,
+      "logps/rejected": -9.440406799316406,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.698478698730469,
+      "rewards/margins": 0.7419284582138062,
+      "rewards/rejected": -9.440406799316406,
+      "semantic_entropy": 0.0022458950988948345,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 15.893608851685936,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": 1.4507125616073608,
+      "logits/rejected": 1.4903157949447632,
+      "logps/chosen": -8.67979907989502,
+      "logps/rejected": -9.233920097351074,
+      "loss": 0.612,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -8.67979907989502,
+      "rewards/margins": 0.5541218519210815,
+      "rewards/rejected": -9.233920097351074,
+      "semantic_entropy": 0.00230026594363153,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 13.347177191718151,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": 1.4876182079315186,
+      "logits/rejected": 1.5341087579727173,
+      "logps/chosen": -8.815591812133789,
+      "logps/rejected": -9.463472366333008,
+      "loss": 0.5524,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.815591812133789,
+      "rewards/margins": 0.6478797197341919,
+      "rewards/rejected": -9.463472366333008,
+      "semantic_entropy": 0.0020592932123690844,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 15.207506373353057,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": 1.4389550685882568,
+      "logits/rejected": 1.5009499788284302,
+      "logps/chosen": -8.99170207977295,
+      "logps/rejected": -9.629568099975586,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.99170207977295,
+      "rewards/margins": 0.6378656625747681,
+      "rewards/rejected": -9.629568099975586,
+      "semantic_entropy": 0.001768794609233737,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 14.442777395060759,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": 1.3885715007781982,
+      "logits/rejected": 1.4695911407470703,
+      "logps/chosen": -9.065951347351074,
+      "logps/rejected": -9.680649757385254,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.065951347351074,
+      "rewards/margins": 0.6146981716156006,
+      "rewards/rejected": -9.680649757385254,
+      "semantic_entropy": 0.0015854325611144304,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 18.786727817319235,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": 1.4829179048538208,
+      "logits/rejected": 1.5331708192825317,
+      "logps/chosen": -8.9779052734375,
+      "logps/rejected": -9.533280372619629,
+      "loss": 0.5952,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -8.9779052734375,
+      "rewards/margins": 0.5553746223449707,
+      "rewards/rejected": -9.533280372619629,
+      "semantic_entropy": 0.0017666567582637072,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 18.661657680767576,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": 1.473697543144226,
+      "logits/rejected": 1.5222572088241577,
+      "logps/chosen": -8.986383438110352,
+      "logps/rejected": -9.580157279968262,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -8.986383438110352,
+      "rewards/margins": 0.5937739610671997,
+      "rewards/rejected": -9.580157279968262,
+      "semantic_entropy": 0.0018506577471271157,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 14.578101176925577,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": 1.4264580011367798,
+      "logits/rejected": 1.480059027671814,
+      "logps/chosen": -8.992395401000977,
+      "logps/rejected": -9.611838340759277,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -8.992395401000977,
+      "rewards/margins": 0.6194430589675903,
+      "rewards/rejected": -9.611838340759277,
+      "semantic_entropy": 0.0018926769262179732,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 14.046279661581393,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": 1.458611011505127,
+      "logits/rejected": 1.4923086166381836,
+      "logps/chosen": -9.0691556930542,
+      "logps/rejected": -9.636763572692871,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -9.0691556930542,
+      "rewards/margins": 0.5676077008247375,
+      "rewards/rejected": -9.636763572692871,
+      "semantic_entropy": 0.0016790159279480577,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 12.088170607146196,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": 1.470457911491394,
+      "logits/rejected": 1.545482873916626,
+      "logps/chosen": -9.171323776245117,
+      "logps/rejected": -9.98707389831543,
+      "loss": 0.4723,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.171323776245117,
+      "rewards/margins": 0.8157499432563782,
+      "rewards/rejected": -9.98707389831543,
+      "semantic_entropy": 0.0014635700499638915,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 19.99463353584722,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": 1.4574049711227417,
+      "logits/rejected": 1.485471487045288,
+      "logps/chosen": -9.051946640014648,
+      "logps/rejected": -9.543763160705566,
+      "loss": 0.5928,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.051946640014648,
+      "rewards/margins": 0.4918156564235687,
+      "rewards/rejected": -9.543763160705566,
+      "semantic_entropy": 0.0017323382198810577,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 20.542947695544992,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": 1.4667167663574219,
+      "logits/rejected": 1.5746221542358398,
+      "logps/chosen": -9.263598442077637,
+      "logps/rejected": -9.88271713256836,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.263598442077637,
+      "rewards/margins": 0.6191186308860779,
+      "rewards/rejected": -9.88271713256836,
+      "semantic_entropy": 0.002059711841866374,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 16.731494233394816,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": 1.446484923362732,
+      "logits/rejected": 1.5131540298461914,
+      "logps/chosen": -9.24215030670166,
+      "logps/rejected": -10.040380477905273,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.24215030670166,
+      "rewards/margins": 0.798229455947876,
+      "rewards/rejected": -10.040380477905273,
+      "semantic_entropy": 0.001486125634983182,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 12.02037592512661,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": 1.4572422504425049,
+      "logits/rejected": 1.5142450332641602,
+      "logps/chosen": -9.1526517868042,
+      "logps/rejected": -9.75566291809082,
+      "loss": 0.5838,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.1526517868042,
+      "rewards/margins": 0.6030122637748718,
+      "rewards/rejected": -9.75566291809082,
+      "semantic_entropy": 0.0016955469036474824,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 37.37482529004528,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": 1.4660760164260864,
+      "logits/rejected": 1.5358116626739502,
+      "logps/chosen": -9.281084060668945,
+      "logps/rejected": -10.09622859954834,
+      "loss": 0.5504,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.281084060668945,
+      "rewards/margins": 0.8151445388793945,
+      "rewards/rejected": -10.09622859954834,
+      "semantic_entropy": 0.0015162710333243012,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 11.916633013854907,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": 1.4610587358474731,
+      "logits/rejected": 1.52230966091156,
+      "logps/chosen": -9.392227172851562,
+      "logps/rejected": -10.135969161987305,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.392227172851562,
+      "rewards/margins": 0.7437421679496765,
+      "rewards/rejected": -10.135969161987305,
+      "semantic_entropy": 0.0013578941579908133,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 14.907780642769412,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": 1.4519511461257935,
+      "logits/rejected": 1.5093753337860107,
+      "logps/chosen": -9.433538436889648,
+      "logps/rejected": -9.938074111938477,
+      "loss": 0.6273,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -9.433538436889648,
+      "rewards/margins": 0.5045345425605774,
+      "rewards/rejected": -9.938074111938477,
+      "semantic_entropy": 0.0012871257495135069,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 17.801696554045368,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": 1.4394404888153076,
+      "logits/rejected": 1.481331706047058,
+      "logps/chosen": -9.300742149353027,
+      "logps/rejected": -9.981010437011719,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.300742149353027,
+      "rewards/margins": 0.6802674531936646,
+      "rewards/rejected": -9.981010437011719,
+      "semantic_entropy": 0.001361246919259429,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 12.634844453916791,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": 1.4397763013839722,
+      "logits/rejected": 1.480520486831665,
+      "logps/chosen": -9.315359115600586,
+      "logps/rejected": -9.943147659301758,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.315359115600586,
+      "rewards/margins": 0.627788782119751,
+      "rewards/rejected": -9.943147659301758,
+      "semantic_entropy": 0.0013189424062147737,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 12.871011471690858,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": 1.417900800704956,
+      "logits/rejected": 1.451626181602478,
+      "logps/chosen": -9.424980163574219,
+      "logps/rejected": -9.942686080932617,
+      "loss": 0.5851,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -9.424980163574219,
+      "rewards/margins": 0.5177065134048462,
+      "rewards/rejected": -9.942686080932617,
+      "semantic_entropy": 0.0011966429883614182,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 12.958238323160362,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": 1.3756444454193115,
+      "logits/rejected": 1.4366334676742554,
+      "logps/chosen": -9.451255798339844,
+      "logps/rejected": -9.890711784362793,
+      "loss": 0.5973,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.451255798339844,
+      "rewards/margins": 0.4394569993019104,
+      "rewards/rejected": -9.890711784362793,
+      "semantic_entropy": 0.00114995997864753,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 15.040823172874564,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": 1.3422110080718994,
+      "logits/rejected": 1.4166682958602905,
+      "logps/chosen": -9.195169448852539,
+      "logps/rejected": -9.850201606750488,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -9.195169448852539,
+      "rewards/margins": 0.655031144618988,
+      "rewards/rejected": -9.850201606750488,
+      "semantic_entropy": 0.0014082149136811495,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 11.896126998337655,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": 1.3955941200256348,
+      "logits/rejected": 1.454282283782959,
+      "logps/chosen": -9.316786766052246,
+      "logps/rejected": -9.962684631347656,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.316786766052246,
+      "rewards/margins": 0.6458979845046997,
+      "rewards/rejected": -9.962684631347656,
+      "semantic_entropy": 0.0013396672438830137,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 26.141780365433526,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": 1.4380090236663818,
+      "logits/rejected": 1.4804362058639526,
+      "logps/chosen": -9.303455352783203,
+      "logps/rejected": -9.74422550201416,
+      "loss": 0.6233,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -9.303455352783203,
+      "rewards/margins": 0.4407702088356018,
+      "rewards/rejected": -9.74422550201416,
+      "semantic_entropy": 0.0013635194627568126,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 13.222918208169155,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": 1.404249906539917,
+      "logits/rejected": 1.460638165473938,
+      "logps/chosen": -9.308123588562012,
+      "logps/rejected": -10.057351112365723,
+      "loss": 0.5493,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.308123588562012,
+      "rewards/margins": 0.7492268085479736,
+      "rewards/rejected": -10.057351112365723,
+      "semantic_entropy": 0.0013175774365663528,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 11.992327272951714,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": 1.3603496551513672,
+      "logits/rejected": 1.415806531906128,
+      "logps/chosen": -9.130047798156738,
+      "logps/rejected": -9.746193885803223,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.130047798156738,
+      "rewards/margins": 0.616145670413971,
+      "rewards/rejected": -9.746193885803223,
+      "semantic_entropy": 0.002505678217858076,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 17.34311772874582,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": 1.323307752609253,
+      "logits/rejected": 1.3809163570404053,
+      "logps/chosen": -9.184372901916504,
+      "logps/rejected": -9.765647888183594,
+      "loss": 0.5621,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.184372901916504,
+      "rewards/margins": 0.581275999546051,
+      "rewards/rejected": -9.765647888183594,
+      "semantic_entropy": 0.0015321519458666444,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 14.342173713566435,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": 1.414975881576538,
+      "logits/rejected": 1.4325586557388306,
+      "logps/chosen": -9.170077323913574,
+      "logps/rejected": -9.831615447998047,
+      "loss": 0.543,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.170077323913574,
+      "rewards/margins": 0.6615375876426697,
+      "rewards/rejected": -9.831615447998047,
+      "semantic_entropy": 0.0017330650007352233,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 10.540170420600656,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": 1.3706494569778442,
+      "logits/rejected": 1.4219094514846802,
+      "logps/chosen": -9.121861457824707,
+      "logps/rejected": -9.771852493286133,
+      "loss": 0.557,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -9.121861457824707,
+      "rewards/margins": 0.6499911546707153,
+      "rewards/rejected": -9.771852493286133,
+      "semantic_entropy": 0.0017241711029782891,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 24.20539474852228,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": 1.4124326705932617,
+      "logits/rejected": 1.5103449821472168,
+      "logps/chosen": -9.120132446289062,
+      "logps/rejected": -9.676496505737305,
+      "loss": 0.568,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.120132446289062,
+      "rewards/margins": 0.5563638806343079,
+      "rewards/rejected": -9.676496505737305,
+      "semantic_entropy": 0.0016992479795590043,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 19.90079200857601,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": 1.4147361516952515,
+      "logits/rejected": 1.4643796682357788,
+      "logps/chosen": -9.107364654541016,
+      "logps/rejected": -9.681783676147461,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -9.107364654541016,
+      "rewards/margins": 0.5744190216064453,
+      "rewards/rejected": -9.681783676147461,
+      "semantic_entropy": 0.0018841770943254232,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 15.920395269714032,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": 1.36264169216156,
+      "logits/rejected": 1.4513592720031738,
+      "logps/chosen": -8.75432300567627,
+      "logps/rejected": -9.375045776367188,
+      "loss": 0.5876,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.75432300567627,
+      "rewards/margins": 0.6207231879234314,
+      "rewards/rejected": -9.375045776367188,
+      "semantic_entropy": 0.0039268904365599155,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 13.662287684823369,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": 1.3365135192871094,
+      "logits/rejected": 1.4371356964111328,
+      "logps/chosen": -8.673608779907227,
+      "logps/rejected": -9.305208206176758,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.673608779907227,
+      "rewards/margins": 0.6316004991531372,
+      "rewards/rejected": -9.305208206176758,
+      "semantic_entropy": 0.002468000864610076,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 12.224935700620788,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": 1.3542077541351318,
+      "logits/rejected": 1.4318268299102783,
+      "logps/chosen": -8.750521659851074,
+      "logps/rejected": -9.267577171325684,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -8.750521659851074,
+      "rewards/margins": 0.5170544385910034,
+      "rewards/rejected": -9.267577171325684,
+      "semantic_entropy": 0.0022069388069212437,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 14.665703210489562,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": 1.3631465435028076,
+      "logits/rejected": 1.3851419687271118,
+      "logps/chosen": -8.816594123840332,
+      "logps/rejected": -9.34947681427002,
+      "loss": 0.6197,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -8.816594123840332,
+      "rewards/margins": 0.5328823924064636,
+      "rewards/rejected": -9.34947681427002,
+      "semantic_entropy": 0.00246635964140296,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 17.958431903260244,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": 1.4095995426177979,
+      "logits/rejected": 1.5218260288238525,
+      "logps/chosen": -9.134257316589355,
+      "logps/rejected": -9.727826118469238,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.134257316589355,
+      "rewards/margins": 0.5935682058334351,
+      "rewards/rejected": -9.727826118469238,
+      "semantic_entropy": 0.0015177963068708777,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 14.78518495907931,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": 1.5228782892227173,
+      "logits/rejected": 1.5653560161590576,
+      "logps/chosen": -9.067325592041016,
+      "logps/rejected": -9.788304328918457,
+      "loss": 0.5578,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.067325592041016,
+      "rewards/margins": 0.7209778428077698,
+      "rewards/rejected": -9.788304328918457,
+      "semantic_entropy": 0.001715084770694375,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 13.626681306820867,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": 1.4755054712295532,
+      "logits/rejected": 1.5510979890823364,
+      "logps/chosen": -9.327742576599121,
+      "logps/rejected": -10.08023738861084,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.327742576599121,
+      "rewards/margins": 0.7524958252906799,
+      "rewards/rejected": -10.08023738861084,
+      "semantic_entropy": 0.0013817392755299807,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 16.242512540785075,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": 1.486583948135376,
+      "logits/rejected": 1.5418224334716797,
+      "logps/chosen": -9.326029777526855,
+      "logps/rejected": -10.055940628051758,
+      "loss": 0.552,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.326029777526855,
+      "rewards/margins": 0.7299106121063232,
+      "rewards/rejected": -10.055940628051758,
+      "semantic_entropy": 0.001374894054606557,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 13.168406624171418,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": 1.4896965026855469,
+      "logits/rejected": 1.5442678928375244,
+      "logps/chosen": -9.355472564697266,
+      "logps/rejected": -9.837401390075684,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -9.355472564697266,
+      "rewards/margins": 0.4819281995296478,
+      "rewards/rejected": -9.837401390075684,
+      "semantic_entropy": 0.0013941603247076273,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 15.559983450748701,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": 1.474988579750061,
+      "logits/rejected": 1.5462048053741455,
+      "logps/chosen": -9.205206871032715,
+      "logps/rejected": -9.69001293182373,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -9.205206871032715,
+      "rewards/margins": 0.484805166721344,
+      "rewards/rejected": -9.69001293182373,
+      "semantic_entropy": 0.002060151193290949,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 12.790684757519099,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": 1.4946880340576172,
+      "logits/rejected": 1.5453252792358398,
+      "logps/chosen": -9.191274642944336,
+      "logps/rejected": -9.782793045043945,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.191274642944336,
+      "rewards/margins": 0.5915166139602661,
+      "rewards/rejected": -9.782793045043945,
+      "semantic_entropy": 0.0015505468472838402,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 13.486221418082918,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": 1.4475295543670654,
+      "logits/rejected": 1.5991915464401245,
+      "logps/chosen": -9.185344696044922,
+      "logps/rejected": -9.985203742980957,
+      "loss": 0.5121,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.185344696044922,
+      "rewards/margins": 0.7998607158660889,
+      "rewards/rejected": -9.985203742980957,
+      "semantic_entropy": 0.0015584348002448678,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 13.050055373388858,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": 1.4835433959960938,
+      "logits/rejected": 1.5183498859405518,
+      "logps/chosen": -9.133069038391113,
+      "logps/rejected": -9.78994083404541,
+      "loss": 0.5566,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -9.133069038391113,
+      "rewards/margins": 0.6568711996078491,
+      "rewards/rejected": -9.78994083404541,
+      "semantic_entropy": 0.0017261076718568802,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 14.795668440297876,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": 1.485611081123352,
+      "logits/rejected": 1.5537751913070679,
+      "logps/chosen": -9.094915390014648,
+      "logps/rejected": -9.79945182800293,
+      "loss": 0.5426,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.094915390014648,
+      "rewards/margins": 0.7045354843139648,
+      "rewards/rejected": -9.79945182800293,
+      "semantic_entropy": 0.0015737510984763503,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 14.912733429205396,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": 1.436413049697876,
+      "logits/rejected": 1.5191280841827393,
+      "logps/chosen": -9.119382858276367,
+      "logps/rejected": -9.738457679748535,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -9.119382858276367,
+      "rewards/margins": 0.61907559633255,
+      "rewards/rejected": -9.738457679748535,
+      "semantic_entropy": 0.0018772294279187918,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 23.58538612522326,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": 1.440748929977417,
+      "logits/rejected": 1.5233433246612549,
+      "logps/chosen": -9.239843368530273,
+      "logps/rejected": -9.746759414672852,
+      "loss": 0.607,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -9.239843368530273,
+      "rewards/margins": 0.5069173574447632,
+      "rewards/rejected": -9.746759414672852,
+      "semantic_entropy": 0.0014747639652341604,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 13.564253428248175,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": 1.402316689491272,
+      "logits/rejected": 1.5004584789276123,
+      "logps/chosen": -9.106497764587402,
+      "logps/rejected": -9.782754898071289,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.106497764587402,
+      "rewards/margins": 0.6762566566467285,
+      "rewards/rejected": -9.782754898071289,
+      "semantic_entropy": 0.0016546149272471666,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 17.0291044633709,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": 1.4681202173233032,
+      "logits/rejected": 1.5597844123840332,
+      "logps/chosen": -9.014376640319824,
+      "logps/rejected": -9.794927597045898,
+      "loss": 0.5671,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.014376640319824,
+      "rewards/margins": 0.7805513143539429,
+      "rewards/rejected": -9.794927597045898,
+      "semantic_entropy": 0.0018271291628479958,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 16.17234226786698,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": 1.451099157333374,
+      "logits/rejected": 1.539367914199829,
+      "logps/chosen": -8.976418495178223,
+      "logps/rejected": -9.617650032043457,
+      "loss": 0.567,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.976418495178223,
+      "rewards/margins": 0.6412309408187866,
+      "rewards/rejected": -9.617650032043457,
+      "semantic_entropy": 0.0018440615385770798,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 14.154941749009216,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": 1.3643250465393066,
+      "logits/rejected": 1.5202983617782593,
+      "logps/chosen": -8.918514251708984,
+      "logps/rejected": -9.851920127868652,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.918514251708984,
+      "rewards/margins": 0.9334057569503784,
+      "rewards/rejected": -9.851920127868652,
+      "semantic_entropy": 0.001965166302397847,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 12.475587010614824,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": 1.3400129079818726,
+      "logits/rejected": 1.4705440998077393,
+      "logps/chosen": -8.580958366394043,
+      "logps/rejected": -9.411206245422363,
+      "loss": 0.5348,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.580958366394043,
+      "rewards/margins": 0.8302468061447144,
+      "rewards/rejected": -9.411206245422363,
+      "semantic_entropy": 0.002642639447003603,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 12.588474060035281,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": 1.3838415145874023,
+      "logits/rejected": 1.4307692050933838,
+      "logps/chosen": -8.608055114746094,
+      "logps/rejected": -9.276836395263672,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.608055114746094,
+      "rewards/margins": 0.6687813997268677,
+      "rewards/rejected": -9.276836395263672,
+      "semantic_entropy": 0.002553506288677454,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 11.790917195494332,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": 1.399780511856079,
+      "logits/rejected": 1.4404734373092651,
+      "logps/chosen": -8.787452697753906,
+      "logps/rejected": -9.384294509887695,
+      "loss": 0.5862,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.787452697753906,
+      "rewards/margins": 0.5968419313430786,
+      "rewards/rejected": -9.384294509887695,
+      "semantic_entropy": 0.0023327034432440996,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 18.81532057496146,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": 1.4533107280731201,
+      "logits/rejected": 1.5081993341445923,
+      "logps/chosen": -9.013750076293945,
+      "logps/rejected": -9.519759178161621,
+      "loss": 0.6249,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.013750076293945,
+      "rewards/margins": 0.5060084462165833,
+      "rewards/rejected": -9.519759178161621,
+      "semantic_entropy": 0.001899567199870944,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 10.918095452049027,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": 1.3967291116714478,
+      "logits/rejected": 1.4510613679885864,
+      "logps/chosen": -8.802974700927734,
+      "logps/rejected": -9.396745681762695,
+      "loss": 0.548,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.802974700927734,
+      "rewards/margins": 0.5937708616256714,
+      "rewards/rejected": -9.396745681762695,
+      "semantic_entropy": 0.0027451002970337868,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 12.220722624049088,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": 1.4750268459320068,
+      "logits/rejected": 1.5838210582733154,
+      "logps/chosen": -9.196192741394043,
+      "logps/rejected": -9.85037899017334,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.196192741394043,
+      "rewards/margins": 0.654186487197876,
+      "rewards/rejected": -9.85037899017334,
+      "semantic_entropy": 0.001516185118816793,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 11.619313552115338,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": 1.4766775369644165,
+      "logits/rejected": 1.5545923709869385,
+      "logps/chosen": -9.073392868041992,
+      "logps/rejected": -9.740924835205078,
+      "loss": 0.553,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.073392868041992,
+      "rewards/margins": 0.6675307154655457,
+      "rewards/rejected": -9.740924835205078,
+      "semantic_entropy": 0.0017678631702437997,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 20.85152340680272,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": 1.4245964288711548,
+      "logits/rejected": 1.4519484043121338,
+      "logps/chosen": -8.933578491210938,
+      "logps/rejected": -9.48484992980957,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -8.933578491210938,
+      "rewards/margins": 0.5512722134590149,
+      "rewards/rejected": -9.48484992980957,
+      "semantic_entropy": 0.0018317581852898002,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 15.479638484425788,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": 1.4926708936691284,
+      "logits/rejected": 1.5244104862213135,
+      "logps/chosen": -9.15540885925293,
+      "logps/rejected": -9.673357009887695,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -9.15540885925293,
+      "rewards/margins": 0.5179484486579895,
+      "rewards/rejected": -9.673357009887695,
+      "semantic_entropy": 0.0015676971524953842,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 11.680271570486585,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": 1.4367287158966064,
+      "logits/rejected": 1.5324538946151733,
+      "logps/chosen": -9.051034927368164,
+      "logps/rejected": -10.024801254272461,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.051034927368164,
+      "rewards/margins": 0.9737651944160461,
+      "rewards/rejected": -10.024801254272461,
+      "semantic_entropy": 0.0018925551557913423,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": 1.6254278421401978,
+      "eval_logits/rejected": 1.6747074127197266,
+      "eval_logps/chosen": -9.115405082702637,
+      "eval_logps/rejected": -9.834186553955078,
+      "eval_loss": 0.5385689735412598,
+      "eval_rewards/accuracies": 0.716617226600647,
+      "eval_rewards/chosen": -9.115405082702637,
+      "eval_rewards/margins": 0.7187818288803101,
+      "eval_rewards/rejected": -9.834186553955078,
+      "eval_runtime": 34.6109,
+      "eval_samples_per_second": 38.861,
+      "eval_semantic_entropy": 0.0016772582894191146,
+      "eval_steps_per_second": 9.737,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 15.099186131984926,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": 1.485991358757019,
+      "logits/rejected": 1.609984040260315,
+      "logps/chosen": -9.191354751586914,
+      "logps/rejected": -9.952881813049316,
+      "loss": 0.5225,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.191354751586914,
+      "rewards/margins": 0.7615259289741516,
+      "rewards/rejected": -9.952881813049316,
+      "semantic_entropy": 0.0015668151900172234,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 11.50535343261545,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": 1.4452701807022095,
+      "logits/rejected": 1.54624342918396,
+      "logps/chosen": -8.940690040588379,
+      "logps/rejected": -9.844390869140625,
+      "loss": 0.4856,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.940690040588379,
+      "rewards/margins": 0.9037016034126282,
+      "rewards/rejected": -9.844390869140625,
+      "semantic_entropy": 0.0020111093763262033,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 20.587289016944442,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": 1.4971530437469482,
+      "logits/rejected": 1.563239336013794,
+      "logps/chosen": -8.949003219604492,
+      "logps/rejected": -9.6390380859375,
+      "loss": 0.6009,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -8.949003219604492,
+      "rewards/margins": 0.6900355219841003,
+      "rewards/rejected": -9.6390380859375,
+      "semantic_entropy": 0.002106769708916545,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 15.32888022294029,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": 1.5007158517837524,
+      "logits/rejected": 1.5934231281280518,
+      "logps/chosen": -9.095519065856934,
+      "logps/rejected": -9.718890190124512,
+      "loss": 0.6117,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -9.095519065856934,
+      "rewards/margins": 0.6233715415000916,
+      "rewards/rejected": -9.718890190124512,
+      "semantic_entropy": 0.001936092390678823,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 15.186453922793774,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": 1.487046718597412,
+      "logits/rejected": 1.514864444732666,
+      "logps/chosen": -9.286542892456055,
+      "logps/rejected": -9.603887557983398,
+      "loss": 0.7021,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -9.286542892456055,
+      "rewards/margins": 0.3173438012599945,
+      "rewards/rejected": -9.603887557983398,
+      "semantic_entropy": 0.0015473442617803812,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 11.476535517553776,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": 1.5084360837936401,
+      "logits/rejected": 1.5790362358093262,
+      "logps/chosen": -9.109070777893066,
+      "logps/rejected": -9.726173400878906,
+      "loss": 0.5716,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.109070777893066,
+      "rewards/margins": 0.6171044111251831,
+      "rewards/rejected": -9.726173400878906,
+      "semantic_entropy": 0.0020177545957267284,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 12.41963247856584,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": 1.490997552871704,
+      "logits/rejected": 1.5654528141021729,
+      "logps/chosen": -9.048565864562988,
+      "logps/rejected": -9.777292251586914,
+      "loss": 0.5288,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.048565864562988,
+      "rewards/margins": 0.7287265062332153,
+      "rewards/rejected": -9.777292251586914,
+      "semantic_entropy": 0.0016265253070741892,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 23.948430790469452,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": 1.5099024772644043,
+      "logits/rejected": 1.5679699182510376,
+      "logps/chosen": -9.202662467956543,
+      "logps/rejected": -9.829662322998047,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.202662467956543,
+      "rewards/margins": 0.6269996762275696,
+      "rewards/rejected": -9.829662322998047,
+      "semantic_entropy": 0.0014407768612727523,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 14.22900997018834,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": 1.520168423652649,
+      "logits/rejected": 1.5453139543533325,
+      "logps/chosen": -9.226568222045898,
+      "logps/rejected": -9.683099746704102,
+      "loss": 0.614,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -9.226568222045898,
+      "rewards/margins": 0.45653238892555237,
+      "rewards/rejected": -9.683099746704102,
+      "semantic_entropy": 0.0016828408697620034,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 19.372840220803972,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": 1.5835860967636108,
+      "logits/rejected": 1.6192537546157837,
+      "logps/chosen": -9.049530982971191,
+      "logps/rejected": -9.765745162963867,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.049530982971191,
+      "rewards/margins": 0.7162145972251892,
+      "rewards/rejected": -9.765745162963867,
+      "semantic_entropy": 0.001774814911186695,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 14.131457825671555,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": 1.5568708181381226,
+      "logits/rejected": 1.615639090538025,
+      "logps/chosen": -9.163629531860352,
+      "logps/rejected": -9.836689949035645,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.163629531860352,
+      "rewards/margins": 0.673060417175293,
+      "rewards/rejected": -9.836689949035645,
+      "semantic_entropy": 0.0015099862357601523,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 16.976861532950906,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": 1.5633503198623657,
+      "logits/rejected": 1.5829474925994873,
+      "logps/chosen": -9.046289443969727,
+      "logps/rejected": -9.727827072143555,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.046289443969727,
+      "rewards/margins": 0.6815365552902222,
+      "rewards/rejected": -9.727827072143555,
+      "semantic_entropy": 0.0017096446827054024,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 14.44348782143371,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": 1.541069507598877,
+      "logits/rejected": 1.5631182193756104,
+      "logps/chosen": -9.116737365722656,
+      "logps/rejected": -9.702676773071289,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.116737365722656,
+      "rewards/margins": 0.5859400033950806,
+      "rewards/rejected": -9.702676773071289,
+      "semantic_entropy": 0.001721767126582563,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 13.489701080503613,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": 1.564348578453064,
+      "logits/rejected": 1.638221025466919,
+      "logps/chosen": -9.015240669250488,
+      "logps/rejected": -9.476325035095215,
+      "loss": 0.6276,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -9.015240669250488,
+      "rewards/margins": 0.4610845148563385,
+      "rewards/rejected": -9.476325035095215,
+      "semantic_entropy": 0.0019704310689121485,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 13.442885015654968,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": 1.5247011184692383,
+      "logits/rejected": 1.6226047277450562,
+      "logps/chosen": -9.011486053466797,
+      "logps/rejected": -9.755830764770508,
+      "loss": 0.5291,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.011486053466797,
+      "rewards/margins": 0.7443450093269348,
+      "rewards/rejected": -9.755830764770508,
+      "semantic_entropy": 0.0019308483460918069,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 21.028257919193422,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": 1.5122172832489014,
+      "logits/rejected": 1.566625714302063,
+      "logps/chosen": -8.935271263122559,
+      "logps/rejected": -9.436363220214844,
+      "loss": 0.6057,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -8.935271263122559,
+      "rewards/margins": 0.501091480255127,
+      "rewards/rejected": -9.436363220214844,
+      "semantic_entropy": 0.0020145003218203783,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 20.73123855147749,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": 1.5335428714752197,
+      "logits/rejected": 1.650514006614685,
+      "logps/chosen": -8.885869979858398,
+      "logps/rejected": -9.548897743225098,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.885869979858398,
+      "rewards/margins": 0.6630276441574097,
+      "rewards/rejected": -9.548897743225098,
+      "semantic_entropy": 0.002077076816931367,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 16.12546960073802,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": 1.539560317993164,
+      "logits/rejected": 1.6516990661621094,
+      "logps/chosen": -8.887602806091309,
+      "logps/rejected": -9.693086624145508,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.887602806091309,
+      "rewards/margins": 0.805484414100647,
+      "rewards/rejected": -9.693086624145508,
+      "semantic_entropy": 0.002023321110755205,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 22.88544790262868,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": 1.4666650295257568,
+      "logits/rejected": 1.552095651626587,
+      "logps/chosen": -8.983027458190918,
+      "logps/rejected": -9.63446044921875,
+      "loss": 0.5374,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.983027458190918,
+      "rewards/margins": 0.6514321565628052,
+      "rewards/rejected": -9.63446044921875,
+      "semantic_entropy": 0.0018566297367215157,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 14.87379537239517,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": 1.5537983179092407,
+      "logits/rejected": 1.5298576354980469,
+      "logps/chosen": -9.263163566589355,
+      "logps/rejected": -9.689915657043457,
+      "loss": 0.6315,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -9.263163566589355,
+      "rewards/margins": 0.42675209045410156,
+      "rewards/rejected": -9.689915657043457,
+      "semantic_entropy": 0.0015242116060107946,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 17.025616916515553,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": 1.5553537607192993,
+      "logits/rejected": 1.6223666667938232,
+      "logps/chosen": -9.29149341583252,
+      "logps/rejected": -10.042890548706055,
+      "loss": 0.5291,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.29149341583252,
+      "rewards/margins": 0.7513971924781799,
+      "rewards/rejected": -10.042890548706055,
+      "semantic_entropy": 0.0013182528782635927,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 13.819516091299423,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": 1.5840507745742798,
+      "logits/rejected": 1.6437841653823853,
+      "logps/chosen": -9.556764602661133,
+      "logps/rejected": -10.10120964050293,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.556764602661133,
+      "rewards/margins": 0.5444464683532715,
+      "rewards/rejected": -10.10120964050293,
+      "semantic_entropy": 0.0012000243877992034,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 21.61944452656483,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": 1.5434563159942627,
+      "logits/rejected": 1.6246392726898193,
+      "logps/chosen": -9.642573356628418,
+      "logps/rejected": -10.251947402954102,
+      "loss": 0.6022,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.642573356628418,
+      "rewards/margins": 0.6093729138374329,
+      "rewards/rejected": -10.251947402954102,
+      "semantic_entropy": 0.0010579716181382537,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 19.196815792109433,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": 1.595031976699829,
+      "logits/rejected": 1.6364631652832031,
+      "logps/chosen": -9.304208755493164,
+      "logps/rejected": -10.011419296264648,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.304208755493164,
+      "rewards/margins": 0.7072099447250366,
+      "rewards/rejected": -10.011419296264648,
+      "semantic_entropy": 0.001661711954511702,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 14.721629563243955,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": 1.5675660371780396,
+      "logits/rejected": 1.6353965997695923,
+      "logps/chosen": -9.172778129577637,
+      "logps/rejected": -10.086724281311035,
+      "loss": 0.4706,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.172778129577637,
+      "rewards/margins": 0.9139444231987,
+      "rewards/rejected": -10.086724281311035,
+      "semantic_entropy": 0.0016364321345463395,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 15.42423023230158,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": 1.5130521059036255,
+      "logits/rejected": 1.5637346506118774,
+      "logps/chosen": -8.946981430053711,
+      "logps/rejected": -9.663934707641602,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.946981430053711,
+      "rewards/margins": 0.7169536352157593,
+      "rewards/rejected": -9.663934707641602,
+      "semantic_entropy": 0.0018817862728610635,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 22.140080895273776,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": 1.4245834350585938,
+      "logits/rejected": 1.5359032154083252,
+      "logps/chosen": -8.910916328430176,
+      "logps/rejected": -9.675212860107422,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -8.910916328430176,
+      "rewards/margins": 0.7642959356307983,
+      "rewards/rejected": -9.675212860107422,
+      "semantic_entropy": 0.002216900233179331,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 19.361655675233937,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": 1.4261760711669922,
+      "logits/rejected": 1.5067189931869507,
+      "logps/chosen": -9.074370384216309,
+      "logps/rejected": -9.644983291625977,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.074370384216309,
+      "rewards/margins": 0.5706132650375366,
+      "rewards/rejected": -9.644983291625977,
+      "semantic_entropy": 0.0018715054029598832,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 14.102435741815867,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": 1.4880585670471191,
+      "logits/rejected": 1.4914236068725586,
+      "logps/chosen": -8.856361389160156,
+      "logps/rejected": -9.490935325622559,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.856361389160156,
+      "rewards/margins": 0.6345736384391785,
+      "rewards/rejected": -9.490935325622559,
+      "semantic_entropy": 0.0021703713573515415,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 19.66445514638333,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": 1.447493314743042,
+      "logits/rejected": 1.4823662042617798,
+      "logps/chosen": -9.06472110748291,
+      "logps/rejected": -9.508506774902344,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -9.06472110748291,
+      "rewards/margins": 0.44378575682640076,
+      "rewards/rejected": -9.508506774902344,
+      "semantic_entropy": 0.0018375074723735452,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 15.224614392558864,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": 1.4635555744171143,
+      "logits/rejected": 1.5294735431671143,
+      "logps/chosen": -9.107048034667969,
+      "logps/rejected": -9.878738403320312,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.107048034667969,
+      "rewards/margins": 0.7716915011405945,
+      "rewards/rejected": -9.878738403320312,
+      "semantic_entropy": 0.001704661874100566,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 16.701138072736953,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": 1.4290311336517334,
+      "logits/rejected": 1.497650384902954,
+      "logps/chosen": -8.861137390136719,
+      "logps/rejected": -9.460371017456055,
+      "loss": 0.559,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.861137390136719,
+      "rewards/margins": 0.5992352366447449,
+      "rewards/rejected": -9.460371017456055,
+      "semantic_entropy": 0.0023012214805930853,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 15.87277876853471,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": 1.4090839624404907,
+      "logits/rejected": 1.4412541389465332,
+      "logps/chosen": -8.945982933044434,
+      "logps/rejected": -9.607915878295898,
+      "loss": 0.5603,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.945982933044434,
+      "rewards/margins": 0.6619333624839783,
+      "rewards/rejected": -9.607915878295898,
+      "semantic_entropy": 0.001860318356193602,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 13.18575838436246,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": 1.3694641590118408,
+      "logits/rejected": 1.4676434993743896,
+      "logps/chosen": -8.695920944213867,
+      "logps/rejected": -9.347196578979492,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.695920944213867,
+      "rewards/margins": 0.6512760519981384,
+      "rewards/rejected": -9.347196578979492,
+      "semantic_entropy": 0.0026350780390203,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 13.286799765890317,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": 1.301949143409729,
+      "logits/rejected": 1.3762426376342773,
+      "logps/chosen": -8.408659934997559,
+      "logps/rejected": -9.12511920928955,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.408659934997559,
+      "rewards/margins": 0.7164583206176758,
+      "rewards/rejected": -9.12511920928955,
+      "semantic_entropy": 0.0038316198624670506,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 17.673002668385454,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": 1.3415143489837646,
+      "logits/rejected": 1.4035823345184326,
+      "logps/chosen": -8.420470237731934,
+      "logps/rejected": -9.168214797973633,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.420470237731934,
+      "rewards/margins": 0.7477445602416992,
+      "rewards/rejected": -9.168214797973633,
+      "semantic_entropy": 0.0032614036463201046,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 14.539106886397272,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": 1.376007080078125,
+      "logits/rejected": 1.4095371961593628,
+      "logps/chosen": -8.568218231201172,
+      "logps/rejected": -9.08269214630127,
+      "loss": 0.5942,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -8.568218231201172,
+      "rewards/margins": 0.5144721269607544,
+      "rewards/rejected": -9.08269214630127,
+      "semantic_entropy": 0.0029874849133193493,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 20.289687764049376,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": 1.3758814334869385,
+      "logits/rejected": 1.4339643716812134,
+      "logps/chosen": -8.567605018615723,
+      "logps/rejected": -9.209396362304688,
+      "loss": 0.5614,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.567605018615723,
+      "rewards/margins": 0.6417905688285828,
+      "rewards/rejected": -9.209396362304688,
+      "semantic_entropy": 0.0029742431361228228,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 11.922032775845617,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": 1.279088020324707,
+      "logits/rejected": 1.3529143333435059,
+      "logps/chosen": -8.297035217285156,
+      "logps/rejected": -9.172525405883789,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.297035217285156,
+      "rewards/margins": 0.8754919767379761,
+      "rewards/rejected": -9.172525405883789,
+      "semantic_entropy": 0.004431852605193853,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 16.383461439036527,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": 1.2708383798599243,
+      "logits/rejected": 1.3168162107467651,
+      "logps/chosen": -8.442758560180664,
+      "logps/rejected": -9.118986129760742,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.442758560180664,
+      "rewards/margins": 0.6762279272079468,
+      "rewards/rejected": -9.118986129760742,
+      "semantic_entropy": 0.0031093729194253683,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 20.359018858336896,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": 1.2447166442871094,
+      "logits/rejected": 1.3163912296295166,
+      "logps/chosen": -8.297564506530762,
+      "logps/rejected": -9.260540962219238,
+      "loss": 0.4484,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.297564506530762,
+      "rewards/margins": 0.962977409362793,
+      "rewards/rejected": -9.260540962219238,
+      "semantic_entropy": 0.0035309146624058485,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 14.234201327436809,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": 1.3074781894683838,
+      "logits/rejected": 1.3886187076568604,
+      "logps/chosen": -8.459022521972656,
+      "logps/rejected": -9.167340278625488,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.459022521972656,
+      "rewards/margins": 0.7083176374435425,
+      "rewards/rejected": -9.167340278625488,
+      "semantic_entropy": 0.00307372841052711,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 15.295645145617597,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": 1.3271987438201904,
+      "logits/rejected": 1.3974753618240356,
+      "logps/chosen": -8.467634201049805,
+      "logps/rejected": -9.220786094665527,
+      "loss": 0.5392,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.467634201049805,
+      "rewards/margins": 0.7531511187553406,
+      "rewards/rejected": -9.220786094665527,
+      "semantic_entropy": 0.0034494749270379543,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 21.7337328826633,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": 1.318862795829773,
+      "logits/rejected": 1.3513762950897217,
+      "logps/chosen": -8.357187271118164,
+      "logps/rejected": -9.029712677001953,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.357187271118164,
+      "rewards/margins": 0.6725241541862488,
+      "rewards/rejected": -9.029712677001953,
+      "semantic_entropy": 0.004945852793753147,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 13.725402172884518,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": 1.3667466640472412,
+      "logits/rejected": 1.4593751430511475,
+      "logps/chosen": -8.342109680175781,
+      "logps/rejected": -9.433480262756348,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.342109680175781,
+      "rewards/margins": 1.0913705825805664,
+      "rewards/rejected": -9.433480262756348,
+      "semantic_entropy": 0.0038740220479667187,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 16.268215866430708,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": 1.4472649097442627,
+      "logits/rejected": 1.4870002269744873,
+      "logps/chosen": -8.670722961425781,
+      "logps/rejected": -9.382451057434082,
+      "loss": 0.5733,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.670722961425781,
+      "rewards/margins": 0.7117277383804321,
+      "rewards/rejected": -9.382451057434082,
+      "semantic_entropy": 0.0024931710213422775,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 18.346092678403597,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": 1.421486735343933,
+      "logits/rejected": 1.4674221277236938,
+      "logps/chosen": -8.757630348205566,
+      "logps/rejected": -9.338727951049805,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.757630348205566,
+      "rewards/margins": 0.5810979008674622,
+      "rewards/rejected": -9.338727951049805,
+      "semantic_entropy": 0.0025094773154705763,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 14.448020403834063,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": 1.4752728939056396,
+      "logits/rejected": 1.5600248575210571,
+      "logps/chosen": -9.012587547302246,
+      "logps/rejected": -9.721729278564453,
+      "loss": 0.5253,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.012587547302246,
+      "rewards/margins": 0.7091428637504578,
+      "rewards/rejected": -9.721729278564453,
+      "semantic_entropy": 0.0018399106338620186,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 15.69177758388867,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": 1.4800866842269897,
+      "logits/rejected": 1.5293830633163452,
+      "logps/chosen": -8.922861099243164,
+      "logps/rejected": -9.655854225158691,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.922861099243164,
+      "rewards/margins": 0.7329918146133423,
+      "rewards/rejected": -9.655854225158691,
+      "semantic_entropy": 0.0018165744841098785,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 15.40946970468777,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": 1.4522178173065186,
+      "logits/rejected": 1.526016116142273,
+      "logps/chosen": -9.169662475585938,
+      "logps/rejected": -9.881834030151367,
+      "loss": 0.5528,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.169662475585938,
+      "rewards/margins": 0.7121713757514954,
+      "rewards/rejected": -9.881834030151367,
+      "semantic_entropy": 0.0014839433133602142,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 14.062550269836313,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": 1.5142227411270142,
+      "logits/rejected": 1.5869578123092651,
+      "logps/chosen": -9.104936599731445,
+      "logps/rejected": -9.97562313079834,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.104936599731445,
+      "rewards/margins": 0.8706881403923035,
+      "rewards/rejected": -9.97562313079834,
+      "semantic_entropy": 0.0018488361965864897,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 15.630668045081045,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": 1.4815324544906616,
+      "logits/rejected": 1.5409051179885864,
+      "logps/chosen": -9.41668701171875,
+      "logps/rejected": -10.191702842712402,
+      "loss": 0.5241,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.41668701171875,
+      "rewards/margins": 0.7750155329704285,
+      "rewards/rejected": -10.191702842712402,
+      "semantic_entropy": 0.001475782715715468,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 13.242687248211777,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": 1.5208501815795898,
+      "logits/rejected": 1.545256495475769,
+      "logps/chosen": -9.402139663696289,
+      "logps/rejected": -10.178632736206055,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -9.402139663696289,
+      "rewards/margins": 0.7764937877655029,
+      "rewards/rejected": -10.178632736206055,
+      "semantic_entropy": 0.0018833735957741737,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 14.860965481141804,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": 1.4982603788375854,
+      "logits/rejected": 1.5713450908660889,
+      "logps/chosen": -9.464235305786133,
+      "logps/rejected": -10.36558723449707,
+      "loss": 0.4869,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.464235305786133,
+      "rewards/margins": 0.901351809501648,
+      "rewards/rejected": -10.36558723449707,
+      "semantic_entropy": 0.0012748087756335735,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 14.848699698829346,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": 1.4869000911712646,
+      "logits/rejected": 1.559282660484314,
+      "logps/chosen": -9.605594635009766,
+      "logps/rejected": -10.228915214538574,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -9.605594635009766,
+      "rewards/margins": 0.6233214735984802,
+      "rewards/rejected": -10.228915214538574,
+      "semantic_entropy": 0.0010930115822702646,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 12.916470951395585,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": 1.5114179849624634,
+      "logits/rejected": 1.5730106830596924,
+      "logps/chosen": -9.405946731567383,
+      "logps/rejected": -10.373247146606445,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.405946731567383,
+      "rewards/margins": 0.9672989845275879,
+      "rewards/rejected": -10.373247146606445,
+      "semantic_entropy": 0.0012646029936149716,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 13.899254013789195,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": 1.476386547088623,
+      "logits/rejected": 1.5120741128921509,
+      "logps/chosen": -9.238429069519043,
+      "logps/rejected": -10.01763916015625,
+      "loss": 0.566,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.238429069519043,
+      "rewards/margins": 0.7792092561721802,
+      "rewards/rejected": -10.01763916015625,
+      "semantic_entropy": 0.0016665862640365958,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 17.694340065290568,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": 1.460688591003418,
+      "logits/rejected": 1.5633126497268677,
+      "logps/chosen": -9.259164810180664,
+      "logps/rejected": -10.112449645996094,
+      "loss": 0.499,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.259164810180664,
+      "rewards/margins": 0.8532848358154297,
+      "rewards/rejected": -10.112449645996094,
+      "semantic_entropy": 0.0014983811415731907,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 13.563621310224423,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": 1.4154044389724731,
+      "logits/rejected": 1.4414846897125244,
+      "logps/chosen": -8.959917068481445,
+      "logps/rejected": -9.712752342224121,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.959917068481445,
+      "rewards/margins": 0.752836287021637,
+      "rewards/rejected": -9.712752342224121,
+      "semantic_entropy": 0.0019731316715478897,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 14.660516947169642,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": 1.3173892498016357,
+      "logits/rejected": 1.4314651489257812,
+      "logps/chosen": -8.96304702758789,
+      "logps/rejected": -9.863996505737305,
+      "loss": 0.5017,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.96304702758789,
+      "rewards/margins": 0.900948703289032,
+      "rewards/rejected": -9.863996505737305,
+      "semantic_entropy": 0.0019585746340453625,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 13.896255533558064,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": 1.3477553129196167,
+      "logits/rejected": 1.4231351613998413,
+      "logps/chosen": -8.810920715332031,
+      "logps/rejected": -9.615228652954102,
+      "loss": 0.5263,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.810920715332031,
+      "rewards/margins": 0.8043081164360046,
+      "rewards/rejected": -9.615228652954102,
+      "semantic_entropy": 0.0023903842084109783,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 11.155597483764682,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": 1.3001863956451416,
+      "logits/rejected": 1.358424186706543,
+      "logps/chosen": -8.788329124450684,
+      "logps/rejected": -9.49779224395752,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.788329124450684,
+      "rewards/margins": 0.7094621658325195,
+      "rewards/rejected": -9.49779224395752,
+      "semantic_entropy": 0.0025005824863910675,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 11.552216433465981,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": 1.3298310041427612,
+      "logits/rejected": 1.405280351638794,
+      "logps/chosen": -8.909482955932617,
+      "logps/rejected": -9.69827938079834,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.909482955932617,
+      "rewards/margins": 0.7887974381446838,
+      "rewards/rejected": -9.69827938079834,
+      "semantic_entropy": 0.0021738801151514053,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 13.089758367379979,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": 1.3830832242965698,
+      "logits/rejected": 1.4127867221832275,
+      "logps/chosen": -8.915755271911621,
+      "logps/rejected": -9.723169326782227,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.915755271911621,
+      "rewards/margins": 0.8074146509170532,
+      "rewards/rejected": -9.723169326782227,
+      "semantic_entropy": 0.002421210054308176,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 21.04079083069253,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": 1.2631875276565552,
+      "logits/rejected": 1.3687554597854614,
+      "logps/chosen": -9.039359092712402,
+      "logps/rejected": -9.78271770477295,
+      "loss": 0.5287,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.039359092712402,
+      "rewards/margins": 0.7433594465255737,
+      "rewards/rejected": -9.78271770477295,
+      "semantic_entropy": 0.0018535302951931953,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 20.259647739594055,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": 1.3131898641586304,
+      "logits/rejected": 1.3920212984085083,
+      "logps/chosen": -8.797829627990723,
+      "logps/rejected": -9.641947746276855,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.797829627990723,
+      "rewards/margins": 0.8441187143325806,
+      "rewards/rejected": -9.641947746276855,
+      "semantic_entropy": 0.0025492773856967688,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 16.749937890515547,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": 1.3014531135559082,
+      "logits/rejected": 1.4167145490646362,
+      "logps/chosen": -8.86495590209961,
+      "logps/rejected": -9.748868942260742,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.86495590209961,
+      "rewards/margins": 0.8839127421379089,
+      "rewards/rejected": -9.748868942260742,
+      "semantic_entropy": 0.002319889608770609,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 14.690166145345525,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": 1.3001452684402466,
+      "logits/rejected": 1.3863813877105713,
+      "logps/chosen": -8.778074264526367,
+      "logps/rejected": -9.854037284851074,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.778074264526367,
+      "rewards/margins": 1.0759637355804443,
+      "rewards/rejected": -9.854037284851074,
+      "semantic_entropy": 0.002366643399000168,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 14.216382631137689,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": 1.3263696432113647,
+      "logits/rejected": 1.423143744468689,
+      "logps/chosen": -9.026573181152344,
+      "logps/rejected": -10.064045906066895,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.026573181152344,
+      "rewards/margins": 1.0374727249145508,
+      "rewards/rejected": -10.064045906066895,
+      "semantic_entropy": 0.0020479513332247734,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 15.074847458009272,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": 1.3452244997024536,
+      "logits/rejected": 1.4783446788787842,
+      "logps/chosen": -9.023907661437988,
+      "logps/rejected": -9.848482131958008,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.023907661437988,
+      "rewards/margins": 0.8245747685432434,
+      "rewards/rejected": -9.848482131958008,
+      "semantic_entropy": 0.0021529803052544594,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 20.291585238714767,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": 1.3832794427871704,
+      "logits/rejected": 1.437805414199829,
+      "logps/chosen": -9.044649124145508,
+      "logps/rejected": -9.84146499633789,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.044649124145508,
+      "rewards/margins": 0.796815812587738,
+      "rewards/rejected": -9.84146499633789,
+      "semantic_entropy": 0.0019277628744021058,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 16.83913813733924,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": 1.3638055324554443,
+      "logits/rejected": 1.3967258930206299,
+      "logps/chosen": -8.895398139953613,
+      "logps/rejected": -9.648693084716797,
+      "loss": 0.5623,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -8.895398139953613,
+      "rewards/margins": 0.7532953023910522,
+      "rewards/rejected": -9.648693084716797,
+      "semantic_entropy": 0.002337618265300989,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 14.794289756389395,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": 1.4123839139938354,
+      "logits/rejected": 1.4693882465362549,
+      "logps/chosen": -9.09030818939209,
+      "logps/rejected": -10.194835662841797,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.09030818939209,
+      "rewards/margins": 1.104527473449707,
+      "rewards/rejected": -10.194835662841797,
+      "semantic_entropy": 0.002133353380486369,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 16.591250393886252,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": 1.378289818763733,
+      "logits/rejected": 1.458984613418579,
+      "logps/chosen": -9.01900863647461,
+      "logps/rejected": -9.77697467803955,
+      "loss": 0.554,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.01900863647461,
+      "rewards/margins": 0.7579656839370728,
+      "rewards/rejected": -9.77697467803955,
+      "semantic_entropy": 0.002121507888659835,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 20.16175036955656,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": 1.458469033241272,
+      "logits/rejected": 1.4859371185302734,
+      "logps/chosen": -8.944389343261719,
+      "logps/rejected": -9.716304779052734,
+      "loss": 0.564,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.944389343261719,
+      "rewards/margins": 0.7719146609306335,
+      "rewards/rejected": -9.716304779052734,
+      "semantic_entropy": 0.0023869164288043976,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 20.337350691934443,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": 1.4423576593399048,
+      "logits/rejected": 1.5021278858184814,
+      "logps/chosen": -9.054642677307129,
+      "logps/rejected": -9.854455947875977,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.054642677307129,
+      "rewards/margins": 0.7998123168945312,
+      "rewards/rejected": -9.854455947875977,
+      "semantic_entropy": 0.002396063879132271,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 19.014883108956756,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": 1.4316773414611816,
+      "logits/rejected": 1.5048754215240479,
+      "logps/chosen": -9.07970905303955,
+      "logps/rejected": -9.822221755981445,
+      "loss": 0.5452,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.07970905303955,
+      "rewards/margins": 0.7425130009651184,
+      "rewards/rejected": -9.822221755981445,
+      "semantic_entropy": 0.002267498755827546,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 19.140189438230284,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": 1.4819996356964111,
+      "logits/rejected": 1.5389657020568848,
+      "logps/chosen": -9.391352653503418,
+      "logps/rejected": -10.000016212463379,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -9.391352653503418,
+      "rewards/margins": 0.6086634397506714,
+      "rewards/rejected": -10.000016212463379,
+      "semantic_entropy": 0.0017153879161924124,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 23.389927195242272,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": 1.5310437679290771,
+      "logits/rejected": 1.610856294631958,
+      "logps/chosen": -9.30865478515625,
+      "logps/rejected": -10.173059463500977,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.30865478515625,
+      "rewards/margins": 0.8644039034843445,
+      "rewards/rejected": -10.173059463500977,
+      "semantic_entropy": 0.0017107215244323015,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 19.07041421998402,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": 1.4543803930282593,
+      "logits/rejected": 1.5376605987548828,
+      "logps/chosen": -9.163732528686523,
+      "logps/rejected": -9.888972282409668,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.163732528686523,
+      "rewards/margins": 0.7252396941184998,
+      "rewards/rejected": -9.888972282409668,
+      "semantic_entropy": 0.0018938692519441247,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": 1.6191567182540894,
+      "eval_logits/rejected": 1.6674200296401978,
+      "eval_logps/chosen": -9.066368103027344,
+      "eval_logps/rejected": -9.809303283691406,
+      "eval_loss": 0.5366692543029785,
+      "eval_rewards/accuracies": 0.715133547782898,
+      "eval_rewards/chosen": -9.066368103027344,
+      "eval_rewards/margins": 0.7429361939430237,
+      "eval_rewards/rejected": -9.809303283691406,
+      "eval_runtime": 34.8929,
+      "eval_samples_per_second": 38.546,
+      "eval_semantic_entropy": 0.0018871984211727977,
+      "eval_steps_per_second": 9.658,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 16.475145640120676,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": 1.5157358646392822,
+      "logits/rejected": 1.5379350185394287,
+      "logps/chosen": -8.964433670043945,
+      "logps/rejected": -9.572263717651367,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -8.964433670043945,
+      "rewards/margins": 0.6078299283981323,
+      "rewards/rejected": -9.572263717651367,
+      "semantic_entropy": 0.002141124801710248,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 16.86515375216242,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": 1.5069200992584229,
+      "logits/rejected": 1.569075345993042,
+      "logps/chosen": -8.834966659545898,
+      "logps/rejected": -9.54367446899414,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.834966659545898,
+      "rewards/margins": 0.708707332611084,
+      "rewards/rejected": -9.54367446899414,
+      "semantic_entropy": 0.0020527788437902927,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 15.354973761979455,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": 1.5185476541519165,
+      "logits/rejected": 1.5808709859848022,
+      "logps/chosen": -8.899877548217773,
+      "logps/rejected": -9.612258911132812,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.899877548217773,
+      "rewards/margins": 0.7123812437057495,
+      "rewards/rejected": -9.612258911132812,
+      "semantic_entropy": 0.002221084199845791,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 18.899908372352115,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": 1.4549216032028198,
+      "logits/rejected": 1.5506192445755005,
+      "logps/chosen": -9.144783020019531,
+      "logps/rejected": -9.960975646972656,
+      "loss": 0.4796,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.144783020019531,
+      "rewards/margins": 0.8161920309066772,
+      "rewards/rejected": -9.960975646972656,
+      "semantic_entropy": 0.0018746672431007028,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 13.914829190624136,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": 1.5008902549743652,
+      "logits/rejected": 1.5811628103256226,
+      "logps/chosen": -8.903162002563477,
+      "logps/rejected": -9.694509506225586,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.903162002563477,
+      "rewards/margins": 0.7913479208946228,
+      "rewards/rejected": -9.694509506225586,
+      "semantic_entropy": 0.0024217250756919384,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 11.945143560926262,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": 1.469326376914978,
+      "logits/rejected": 1.493141770362854,
+      "logps/chosen": -8.683900833129883,
+      "logps/rejected": -9.488600730895996,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.683900833129883,
+      "rewards/margins": 0.8047006726264954,
+      "rewards/rejected": -9.488600730895996,
+      "semantic_entropy": 0.002543060341849923,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 18.48649056544402,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": 1.434413194656372,
+      "logits/rejected": 1.5085570812225342,
+      "logps/chosen": -8.859766006469727,
+      "logps/rejected": -9.59799575805664,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.859766006469727,
+      "rewards/margins": 0.7382292747497559,
+      "rewards/rejected": -9.59799575805664,
+      "semantic_entropy": 0.0027851208578795195,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 21.810229679653766,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": 1.409227967262268,
+      "logits/rejected": 1.4552608728408813,
+      "logps/chosen": -8.750555992126465,
+      "logps/rejected": -9.713554382324219,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.750555992126465,
+      "rewards/margins": 0.962997317314148,
+      "rewards/rejected": -9.713554382324219,
+      "semantic_entropy": 0.0025330062489956617,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 17.51049940444659,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": 1.5057343244552612,
+      "logits/rejected": 1.5392143726348877,
+      "logps/chosen": -8.969651222229004,
+      "logps/rejected": -9.774197578430176,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.969651222229004,
+      "rewards/margins": 0.8045468330383301,
+      "rewards/rejected": -9.774197578430176,
+      "semantic_entropy": 0.0022439986933022738,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 21.199837224617394,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": 1.4616109132766724,
+      "logits/rejected": 1.568414568901062,
+      "logps/chosen": -8.820857048034668,
+      "logps/rejected": -9.691858291625977,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.820857048034668,
+      "rewards/margins": 0.8710017204284668,
+      "rewards/rejected": -9.691858291625977,
+      "semantic_entropy": 0.002310030162334442,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 16.669852605118784,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": 1.4513351917266846,
+      "logits/rejected": 1.5431009531021118,
+      "logps/chosen": -8.731821060180664,
+      "logps/rejected": -9.787078857421875,
+      "loss": 0.4867,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.731821060180664,
+      "rewards/margins": 1.0552575588226318,
+      "rewards/rejected": -9.787078857421875,
+      "semantic_entropy": 0.003120484296232462,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 21.97584965325225,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": 1.5134875774383545,
+      "logits/rejected": 1.5621006488800049,
+      "logps/chosen": -8.71433162689209,
+      "logps/rejected": -9.653242111206055,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.71433162689209,
+      "rewards/margins": 0.9389097094535828,
+      "rewards/rejected": -9.653242111206055,
+      "semantic_entropy": 0.0032497632782906294,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 24.026742441481414,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": 1.4807829856872559,
+      "logits/rejected": 1.5714401006698608,
+      "logps/chosen": -8.859692573547363,
+      "logps/rejected": -9.832815170288086,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.859692573547363,
+      "rewards/margins": 0.9731229543685913,
+      "rewards/rejected": -9.832815170288086,
+      "semantic_entropy": 0.002450957428663969,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 16.113962274396382,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": 1.4546705484390259,
+      "logits/rejected": 1.55166757106781,
+      "logps/chosen": -8.585235595703125,
+      "logps/rejected": -9.709646224975586,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -8.585235595703125,
+      "rewards/margins": 1.1244105100631714,
+      "rewards/rejected": -9.709646224975586,
+      "semantic_entropy": 0.003091959049925208,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 22.450776290964146,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": 1.449492335319519,
+      "logits/rejected": 1.524114727973938,
+      "logps/chosen": -8.611242294311523,
+      "logps/rejected": -9.478927612304688,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.611242294311523,
+      "rewards/margins": 0.8676840662956238,
+      "rewards/rejected": -9.478927612304688,
+      "semantic_entropy": 0.003036620793864131,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 18.685932790587415,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": 1.3794857263565063,
+      "logits/rejected": 1.4855682849884033,
+      "logps/chosen": -8.423791885375977,
+      "logps/rejected": -9.371033668518066,
+      "loss": 0.5089,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.423791885375977,
+      "rewards/margins": 0.9472408294677734,
+      "rewards/rejected": -9.371033668518066,
+      "semantic_entropy": 0.0036808927543461323,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 24.343619523158207,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": 1.4342999458312988,
+      "logits/rejected": 1.4923713207244873,
+      "logps/chosen": -8.48465347290039,
+      "logps/rejected": -9.59015941619873,
+      "loss": 0.4758,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -8.48465347290039,
+      "rewards/margins": 1.1055054664611816,
+      "rewards/rejected": -9.59015941619873,
+      "semantic_entropy": 0.0031126588582992554,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 20.977509941209032,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": 1.4219400882720947,
+      "logits/rejected": 1.4429560899734497,
+      "logps/chosen": -8.357909202575684,
+      "logps/rejected": -9.132102012634277,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.357909202575684,
+      "rewards/margins": 0.7741910815238953,
+      "rewards/rejected": -9.132102012634277,
+      "semantic_entropy": 0.004425314255058765,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 18.26046631692716,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": 1.4243085384368896,
+      "logits/rejected": 1.5209802389144897,
+      "logps/chosen": -8.611546516418457,
+      "logps/rejected": -9.455183982849121,
+      "loss": 0.5246,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.611546516418457,
+      "rewards/margins": 0.8436384201049805,
+      "rewards/rejected": -9.455183982849121,
+      "semantic_entropy": 0.003522956743836403,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 23.121433631538338,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": 1.501503348350525,
+      "logits/rejected": 1.5335972309112549,
+      "logps/chosen": -8.632120132446289,
+      "logps/rejected": -9.486554145812988,
+      "loss": 0.5202,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.632120132446289,
+      "rewards/margins": 0.8544337153434753,
+      "rewards/rejected": -9.486554145812988,
+      "semantic_entropy": 0.003261714708060026,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 18.677802973617542,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": 1.5275928974151611,
+      "logits/rejected": 1.621813178062439,
+      "logps/chosen": -8.672503471374512,
+      "logps/rejected": -9.728543281555176,
+      "loss": 0.4567,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -8.672503471374512,
+      "rewards/margins": 1.0560390949249268,
+      "rewards/rejected": -9.728543281555176,
+      "semantic_entropy": 0.0024845425505191088,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 17.214451105130532,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": 1.5551400184631348,
+      "logits/rejected": 1.6167933940887451,
+      "logps/chosen": -9.005258560180664,
+      "logps/rejected": -10.04313850402832,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.005258560180664,
+      "rewards/margins": 1.0378782749176025,
+      "rewards/rejected": -10.04313850402832,
+      "semantic_entropy": 0.001985263079404831,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 19.049223890329166,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": 1.5736329555511475,
+      "logits/rejected": 1.6078697443008423,
+      "logps/chosen": -9.118875503540039,
+      "logps/rejected": -10.01548957824707,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.118875503540039,
+      "rewards/margins": 0.8966142535209656,
+      "rewards/rejected": -10.01548957824707,
+      "semantic_entropy": 0.002023695968091488,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 17.678719830186363,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": 1.5543054342269897,
+      "logits/rejected": 1.587915062904358,
+      "logps/chosen": -8.958490371704102,
+      "logps/rejected": -9.75609302520752,
+      "loss": 0.5168,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.958490371704102,
+      "rewards/margins": 0.7976015210151672,
+      "rewards/rejected": -9.75609302520752,
+      "semantic_entropy": 0.0025089250411838293,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 16.819713760678116,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": 1.53280770778656,
+      "logits/rejected": 1.5885450839996338,
+      "logps/chosen": -9.271095275878906,
+      "logps/rejected": -10.186058044433594,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.271095275878906,
+      "rewards/margins": 0.914962887763977,
+      "rewards/rejected": -10.186058044433594,
+      "semantic_entropy": 0.0019916072487831116,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 19.562720614980087,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": 1.5585497617721558,
+      "logits/rejected": 1.6728546619415283,
+      "logps/chosen": -9.097219467163086,
+      "logps/rejected": -10.13254165649414,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.097219467163086,
+      "rewards/margins": 1.0353223085403442,
+      "rewards/rejected": -10.13254165649414,
+      "semantic_entropy": 0.002022890839725733,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 21.913191709975568,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": 1.4773579835891724,
+      "logits/rejected": 1.5822582244873047,
+      "logps/chosen": -8.943025588989258,
+      "logps/rejected": -9.998074531555176,
+      "loss": 0.4976,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.943025588989258,
+      "rewards/margins": 1.0550485849380493,
+      "rewards/rejected": -9.998074531555176,
+      "semantic_entropy": 0.0024692413862794638,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 24.734715402188808,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": 1.4927732944488525,
+      "logits/rejected": 1.5666470527648926,
+      "logps/chosen": -8.69700813293457,
+      "logps/rejected": -9.660484313964844,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.69700813293457,
+      "rewards/margins": 0.9634763598442078,
+      "rewards/rejected": -9.660484313964844,
+      "semantic_entropy": 0.002958224155008793,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 19.315329365380975,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": 1.4899407625198364,
+      "logits/rejected": 1.5592448711395264,
+      "logps/chosen": -8.632023811340332,
+      "logps/rejected": -9.904651641845703,
+      "loss": 0.4649,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.632023811340332,
+      "rewards/margins": 1.2726271152496338,
+      "rewards/rejected": -9.904651641845703,
+      "semantic_entropy": 0.003169430885463953,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 21.556857879584634,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": 1.4425344467163086,
+      "logits/rejected": 1.4999274015426636,
+      "logps/chosen": -8.734537124633789,
+      "logps/rejected": -9.683516502380371,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.734537124633789,
+      "rewards/margins": 0.9489782452583313,
+      "rewards/rejected": -9.683516502380371,
+      "semantic_entropy": 0.002640213118866086,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 21.252899279342234,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": 1.4922435283660889,
+      "logits/rejected": 1.5495948791503906,
+      "logps/chosen": -8.934083938598633,
+      "logps/rejected": -9.567415237426758,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -8.934083938598633,
+      "rewards/margins": 0.6333316564559937,
+      "rewards/rejected": -9.567415237426758,
+      "semantic_entropy": 0.0021444126032292843,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 20.615974709216026,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": 1.5486667156219482,
+      "logits/rejected": 1.617546796798706,
+      "logps/chosen": -8.919658660888672,
+      "logps/rejected": -9.936529159545898,
+      "loss": 0.506,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.919658660888672,
+      "rewards/margins": 1.0168721675872803,
+      "rewards/rejected": -9.936529159545898,
+      "semantic_entropy": 0.002491950523108244,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 17.06966737640149,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": 1.5858627557754517,
+      "logits/rejected": 1.692470908164978,
+      "logps/chosen": -9.182917594909668,
+      "logps/rejected": -10.162888526916504,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.182917594909668,
+      "rewards/margins": 0.9799696803092957,
+      "rewards/rejected": -10.162888526916504,
+      "semantic_entropy": 0.0018209556583315134,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 12.691749456525942,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": 1.5960538387298584,
+      "logits/rejected": 1.66008722782135,
+      "logps/chosen": -9.286748886108398,
+      "logps/rejected": -10.225370407104492,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.286748886108398,
+      "rewards/margins": 0.9386210441589355,
+      "rewards/rejected": -10.225370407104492,
+      "semantic_entropy": 0.0015730417799204588,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 12.385115013142642,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": 1.5959428548812866,
+      "logits/rejected": 1.6947917938232422,
+      "logps/chosen": -9.423731803894043,
+      "logps/rejected": -10.234014511108398,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.423731803894043,
+      "rewards/margins": 0.8102821111679077,
+      "rewards/rejected": -10.234014511108398,
+      "semantic_entropy": 0.0012895747786387801,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 14.002487410393528,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": 1.6069819927215576,
+      "logits/rejected": 1.65618097782135,
+      "logps/chosen": -9.559354782104492,
+      "logps/rejected": -10.584454536437988,
+      "loss": 0.4408,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.559354782104492,
+      "rewards/margins": 1.0250996351242065,
+      "rewards/rejected": -10.584454536437988,
+      "semantic_entropy": 0.001201879233121872,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 18.002903261105022,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": 1.6735864877700806,
+      "logits/rejected": 1.7481311559677124,
+      "logps/chosen": -9.643770217895508,
+      "logps/rejected": -10.39100456237793,
+      "loss": 0.5238,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.643770217895508,
+      "rewards/margins": 0.7472354769706726,
+      "rewards/rejected": -10.39100456237793,
+      "semantic_entropy": 0.0010812152177095413,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 19.421449954660613,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": 1.6535457372665405,
+      "logits/rejected": 1.679711103439331,
+      "logps/chosen": -9.444608688354492,
+      "logps/rejected": -10.370214462280273,
+      "loss": 0.4819,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.444608688354492,
+      "rewards/margins": 0.9256056547164917,
+      "rewards/rejected": -10.370214462280273,
+      "semantic_entropy": 0.0013431215193122625,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 15.804053481674199,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": 1.5914204120635986,
+      "logits/rejected": 1.6604562997817993,
+      "logps/chosen": -9.34300708770752,
+      "logps/rejected": -10.218287467956543,
+      "loss": 0.5486,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -9.34300708770752,
+      "rewards/margins": 0.8752802610397339,
+      "rewards/rejected": -10.218287467956543,
+      "semantic_entropy": 0.0017238326836377382,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 14.817452019364227,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": 1.6412147283554077,
+      "logits/rejected": 1.6488441228866577,
+      "logps/chosen": -9.274812698364258,
+      "logps/rejected": -10.119780540466309,
+      "loss": 0.5139,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.274812698364258,
+      "rewards/margins": 0.8449678421020508,
+      "rewards/rejected": -10.119780540466309,
+      "semantic_entropy": 0.0015831742202863097,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 16.507602351610547,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": 1.5910123586654663,
+      "logits/rejected": 1.684282660484314,
+      "logps/chosen": -9.40357780456543,
+      "logps/rejected": -10.202895164489746,
+      "loss": 0.5276,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.40357780456543,
+      "rewards/margins": 0.7993172407150269,
+      "rewards/rejected": -10.202895164489746,
+      "semantic_entropy": 0.0015228064730763435,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 19.311704189522736,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": 1.5643151998519897,
+      "logits/rejected": 1.6250832080841064,
+      "logps/chosen": -8.957685470581055,
+      "logps/rejected": -9.986357688903809,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.957685470581055,
+      "rewards/margins": 1.028670310974121,
+      "rewards/rejected": -9.986357688903809,
+      "semantic_entropy": 0.0022164848633110523,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 16.257960734722058,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": 1.554520845413208,
+      "logits/rejected": 1.5907957553863525,
+      "logps/chosen": -8.964838981628418,
+      "logps/rejected": -9.805173873901367,
+      "loss": 0.5196,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.964838981628418,
+      "rewards/margins": 0.8403347134590149,
+      "rewards/rejected": -9.805173873901367,
+      "semantic_entropy": 0.0020810826681554317,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 20.558162856618736,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": 1.4920105934143066,
+      "logits/rejected": 1.5562220811843872,
+      "logps/chosen": -8.707345962524414,
+      "logps/rejected": -9.551727294921875,
+      "loss": 0.5362,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.707345962524414,
+      "rewards/margins": 0.8443810343742371,
+      "rewards/rejected": -9.551727294921875,
+      "semantic_entropy": 0.0024511627852916718,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 14.640034653132735,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": 1.4889882802963257,
+      "logits/rejected": 1.6415224075317383,
+      "logps/chosen": -8.724967956542969,
+      "logps/rejected": -9.829259872436523,
+      "loss": 0.4534,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.724967956542969,
+      "rewards/margins": 1.1042922735214233,
+      "rewards/rejected": -9.829259872436523,
+      "semantic_entropy": 0.0025597529020160437,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 16.16606565677445,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": 1.5296841859817505,
+      "logits/rejected": 1.639047384262085,
+      "logps/chosen": -8.557111740112305,
+      "logps/rejected": -9.632044792175293,
+      "loss": 0.4837,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.557111740112305,
+      "rewards/margins": 1.0749335289001465,
+      "rewards/rejected": -9.632044792175293,
+      "semantic_entropy": 0.002739387098699808,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 21.65653534747905,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": 1.5393953323364258,
+      "logits/rejected": 1.6439985036849976,
+      "logps/chosen": -8.793755531311035,
+      "logps/rejected": -9.804680824279785,
+      "loss": 0.4856,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.793755531311035,
+      "rewards/margins": 1.010925054550171,
+      "rewards/rejected": -9.804680824279785,
+      "semantic_entropy": 0.0023210537619888783,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 19.241643183402633,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": 1.573004126548767,
+      "logits/rejected": 1.6888549327850342,
+      "logps/chosen": -8.829145431518555,
+      "logps/rejected": -9.769402503967285,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.829145431518555,
+      "rewards/margins": 0.9402561187744141,
+      "rewards/rejected": -9.769402503967285,
+      "semantic_entropy": 0.002507644472643733,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 19.987865206472758,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": 1.646235466003418,
+      "logits/rejected": 1.668224573135376,
+      "logps/chosen": -8.805196762084961,
+      "logps/rejected": -9.755497932434082,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.805196762084961,
+      "rewards/margins": 0.9503012895584106,
+      "rewards/rejected": -9.755497932434082,
+      "semantic_entropy": 0.0024248571135103703,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 18.168802593004767,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": 1.613142967224121,
+      "logits/rejected": 1.6410911083221436,
+      "logps/chosen": -9.034639358520508,
+      "logps/rejected": -9.951902389526367,
+      "loss": 0.5253,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.034639358520508,
+      "rewards/margins": 0.9172632098197937,
+      "rewards/rejected": -9.951902389526367,
+      "semantic_entropy": 0.0020687184296548367,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 26.422478045706164,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": 1.4938730001449585,
+      "logits/rejected": 1.5289428234100342,
+      "logps/chosen": -9.047218322753906,
+      "logps/rejected": -9.899124145507812,
+      "loss": 0.5481,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.047218322753906,
+      "rewards/margins": 0.851905345916748,
+      "rewards/rejected": -9.899124145507812,
+      "semantic_entropy": 0.00187408528290689,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 17.751510743185502,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": 1.5519033670425415,
+      "logits/rejected": 1.5536390542984009,
+      "logps/chosen": -8.834941864013672,
+      "logps/rejected": -9.552136421203613,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.834941864013672,
+      "rewards/margins": 0.7171937823295593,
+      "rewards/rejected": -9.552136421203613,
+      "semantic_entropy": 0.002630439819768071,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 16.59260046116158,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": 1.4763872623443604,
+      "logits/rejected": 1.520146369934082,
+      "logps/chosen": -8.775238037109375,
+      "logps/rejected": -9.679451942443848,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.775238037109375,
+      "rewards/margins": 0.904212474822998,
+      "rewards/rejected": -9.679451942443848,
+      "semantic_entropy": 0.0026751295663416386,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 20.93732342739116,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": 1.4435828924179077,
+      "logits/rejected": 1.535719394683838,
+      "logps/chosen": -8.82971477508545,
+      "logps/rejected": -9.890266418457031,
+      "loss": 0.4516,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -8.82971477508545,
+      "rewards/margins": 1.0605512857437134,
+      "rewards/rejected": -9.890266418457031,
+      "semantic_entropy": 0.0025975543539971113,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 14.884703038683332,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": 1.457758903503418,
+      "logits/rejected": 1.5497058629989624,
+      "logps/chosen": -8.773674011230469,
+      "logps/rejected": -9.963736534118652,
+      "loss": 0.422,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -8.773674011230469,
+      "rewards/margins": 1.1900625228881836,
+      "rewards/rejected": -9.963736534118652,
+      "semantic_entropy": 0.0027308736462146044,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 23.288497818004796,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": 1.4490413665771484,
+      "logits/rejected": 1.5511395931243896,
+      "logps/chosen": -8.670270919799805,
+      "logps/rejected": -9.593199729919434,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -8.670270919799805,
+      "rewards/margins": 0.9229286909103394,
+      "rewards/rejected": -9.593199729919434,
+      "semantic_entropy": 0.0027645898517221212,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 12.977927423331247,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": 1.506761908531189,
+      "logits/rejected": 1.625748872756958,
+      "logps/chosen": -8.787134170532227,
+      "logps/rejected": -9.822595596313477,
+      "loss": 0.4667,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.787134170532227,
+      "rewards/margins": 1.0354615449905396,
+      "rewards/rejected": -9.822595596313477,
+      "semantic_entropy": 0.0025216182693839073,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 20.018286935831018,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": 1.49281907081604,
+      "logits/rejected": 1.589317798614502,
+      "logps/chosen": -8.790802001953125,
+      "logps/rejected": -9.768667221069336,
+      "loss": 0.5211,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.790802001953125,
+      "rewards/margins": 0.9778652191162109,
+      "rewards/rejected": -9.768667221069336,
+      "semantic_entropy": 0.002556472085416317,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 20.239076290281115,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": 1.5663044452667236,
+      "logits/rejected": 1.6217765808105469,
+      "logps/chosen": -9.124322891235352,
+      "logps/rejected": -9.920831680297852,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.124322891235352,
+      "rewards/margins": 0.796509325504303,
+      "rewards/rejected": -9.920831680297852,
+      "semantic_entropy": 0.0021244939416646957,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 21.85066226650172,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": 1.5925066471099854,
+      "logits/rejected": 1.6792795658111572,
+      "logps/chosen": -9.045327186584473,
+      "logps/rejected": -10.013941764831543,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.045327186584473,
+      "rewards/margins": 0.9686155319213867,
+      "rewards/rejected": -10.013941764831543,
+      "semantic_entropy": 0.0018998641753569245,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 18.884264379532695,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": 1.5862023830413818,
+      "logits/rejected": 1.653180718421936,
+      "logps/chosen": -9.409761428833008,
+      "logps/rejected": -10.329679489135742,
+      "loss": 0.5096,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.409761428833008,
+      "rewards/margins": 0.9199188947677612,
+      "rewards/rejected": -10.329679489135742,
+      "semantic_entropy": 0.001583190867677331,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 26.427801945825692,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": 1.606117844581604,
+      "logits/rejected": 1.7403303384780884,
+      "logps/chosen": -9.203458786010742,
+      "logps/rejected": -10.320809364318848,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.203458786010742,
+      "rewards/margins": 1.1173508167266846,
+      "rewards/rejected": -10.320809364318848,
+      "semantic_entropy": 0.0025080505292862654,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 16.279249645594486,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": 1.5619335174560547,
+      "logits/rejected": 1.6787865161895752,
+      "logps/chosen": -9.294695854187012,
+      "logps/rejected": -10.163084030151367,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.294695854187012,
+      "rewards/margins": 0.8683876991271973,
+      "rewards/rejected": -10.163084030151367,
+      "semantic_entropy": 0.0017189240315929055,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 17.482112337624685,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": 1.5842080116271973,
+      "logits/rejected": 1.624611258506775,
+      "logps/chosen": -8.984404563903809,
+      "logps/rejected": -9.932075500488281,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.984404563903809,
+      "rewards/margins": 0.9476703405380249,
+      "rewards/rejected": -9.932075500488281,
+      "semantic_entropy": 0.0019870884716510773,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 19.510908501039093,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": 1.5922536849975586,
+      "logits/rejected": 1.625558853149414,
+      "logps/chosen": -8.55893611907959,
+      "logps/rejected": -9.651240348815918,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -8.55893611907959,
+      "rewards/margins": 1.0923033952713013,
+      "rewards/rejected": -9.651240348815918,
+      "semantic_entropy": 0.003535256255418062,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 21.749831372650515,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": 1.4513466358184814,
+      "logits/rejected": 1.533745527267456,
+      "logps/chosen": -8.3834810256958,
+      "logps/rejected": -9.366899490356445,
+      "loss": 0.5311,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -8.3834810256958,
+      "rewards/margins": 0.9834175109863281,
+      "rewards/rejected": -9.366899490356445,
+      "semantic_entropy": 0.0039050192572176456,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 15.926623256320134,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": 1.4681706428527832,
+      "logits/rejected": 1.5388247966766357,
+      "logps/chosen": -8.436796188354492,
+      "logps/rejected": -9.487807273864746,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.436796188354492,
+      "rewards/margins": 1.0510127544403076,
+      "rewards/rejected": -9.487807273864746,
+      "semantic_entropy": 0.0033696871250867844,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 17.209530274868904,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": 1.4030472040176392,
+      "logits/rejected": 1.5322940349578857,
+      "logps/chosen": -8.414661407470703,
+      "logps/rejected": -9.389867782592773,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.414661407470703,
+      "rewards/margins": 0.9752070307731628,
+      "rewards/rejected": -9.389867782592773,
+      "semantic_entropy": 0.0032310497481375933,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 16.612240220916235,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": 1.471640706062317,
+      "logits/rejected": 1.5845543146133423,
+      "logps/chosen": -8.382547378540039,
+      "logps/rejected": -9.503714561462402,
+      "loss": 0.4726,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.382547378540039,
+      "rewards/margins": 1.1211678981781006,
+      "rewards/rejected": -9.503714561462402,
+      "semantic_entropy": 0.003393827471882105,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 16.780139037068473,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": 1.495492696762085,
+      "logits/rejected": 1.609013319015503,
+      "logps/chosen": -8.577532768249512,
+      "logps/rejected": -9.539929389953613,
+      "loss": 0.4746,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.577532768249512,
+      "rewards/margins": 0.9623967409133911,
+      "rewards/rejected": -9.539929389953613,
+      "semantic_entropy": 0.0030008340254426003,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 21.23814827730485,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": 1.527411699295044,
+      "logits/rejected": 1.621955156326294,
+      "logps/chosen": -8.546480178833008,
+      "logps/rejected": -9.673379898071289,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -8.546480178833008,
+      "rewards/margins": 1.1268999576568604,
+      "rewards/rejected": -9.673379898071289,
+      "semantic_entropy": 0.003136562881991267,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 21.957486629032786,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": 1.5210490226745605,
+      "logits/rejected": 1.603949785232544,
+      "logps/chosen": -8.890775680541992,
+      "logps/rejected": -9.885618209838867,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.890775680541992,
+      "rewards/margins": 0.994842529296875,
+      "rewards/rejected": -9.885618209838867,
+      "semantic_entropy": 0.00256163883022964,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 24.662438091416348,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": 1.5244336128234863,
+      "logits/rejected": 1.606689453125,
+      "logps/chosen": -8.923205375671387,
+      "logps/rejected": -9.928789138793945,
+      "loss": 0.5198,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.923205375671387,
+      "rewards/margins": 1.0055832862854004,
+      "rewards/rejected": -9.928789138793945,
+      "semantic_entropy": 0.0031096714083105326,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 22.10503084133697,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": 1.565510630607605,
+      "logits/rejected": 1.6595823764801025,
+      "logps/chosen": -9.140008926391602,
+      "logps/rejected": -10.118307113647461,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.140008926391602,
+      "rewards/margins": 0.9782990217208862,
+      "rewards/rejected": -10.118307113647461,
+      "semantic_entropy": 0.002083156257867813,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 21.768174245123163,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": 1.5303257703781128,
+      "logits/rejected": 1.6365220546722412,
+      "logps/chosen": -8.971370697021484,
+      "logps/rejected": -9.918008804321289,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.971370697021484,
+      "rewards/margins": 0.9466397166252136,
+      "rewards/rejected": -9.918008804321289,
+      "semantic_entropy": 0.0021284637041389942,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 25.00258827263837,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": 1.538752555847168,
+      "logits/rejected": 1.6539390087127686,
+      "logps/chosen": -8.950407028198242,
+      "logps/rejected": -10.092008590698242,
+      "loss": 0.4927,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.950407028198242,
+      "rewards/margins": 1.1416021585464478,
+      "rewards/rejected": -10.092008590698242,
+      "semantic_entropy": 0.0025987981352955103,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 22.545377926738666,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": 1.577199935913086,
+      "logits/rejected": 1.6635043621063232,
+      "logps/chosen": -9.149799346923828,
+      "logps/rejected": -10.218670845031738,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.149799346923828,
+      "rewards/margins": 1.0688717365264893,
+      "rewards/rejected": -10.218670845031738,
+      "semantic_entropy": 0.0022483100183308125,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 19.905271304310993,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": 1.6127588748931885,
+      "logits/rejected": 1.6536674499511719,
+      "logps/chosen": -9.080711364746094,
+      "logps/rejected": -10.294976234436035,
+      "loss": 0.4443,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.080711364746094,
+      "rewards/margins": 1.214263677597046,
+      "rewards/rejected": -10.294976234436035,
+      "semantic_entropy": 0.0020660809241235256,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 27.01168989709476,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": 1.4923503398895264,
+      "logits/rejected": 1.5611363649368286,
+      "logps/chosen": -9.268001556396484,
+      "logps/rejected": -10.1087646484375,
+      "loss": 0.521,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.268001556396484,
+      "rewards/margins": 0.8407636880874634,
+      "rewards/rejected": -10.1087646484375,
+      "semantic_entropy": 0.001690432196483016,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 21.1805492184962,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": 1.5662306547164917,
+      "logits/rejected": 1.6370054483413696,
+      "logps/chosen": -9.073115348815918,
+      "logps/rejected": -9.98923110961914,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.073115348815918,
+      "rewards/margins": 0.9161149859428406,
+      "rewards/rejected": -9.98923110961914,
+      "semantic_entropy": 0.0020183429587632418,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": 1.7479557991027832,
+      "eval_logits/rejected": 1.8155590295791626,
+      "eval_logps/chosen": -9.271117210388184,
+      "eval_logps/rejected": -10.26412582397461,
+      "eval_loss": 0.5360136032104492,
+      "eval_rewards/accuracies": 0.7307121753692627,
+      "eval_rewards/chosen": -9.271117210388184,
+      "eval_rewards/margins": 0.9930074214935303,
+      "eval_rewards/rejected": -10.26412582397461,
+      "eval_runtime": 34.6786,
+      "eval_samples_per_second": 38.785,
+      "eval_semantic_entropy": 0.0018405403243377805,
+      "eval_steps_per_second": 9.718,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 14.88877333349088,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": 1.5526891946792603,
+      "logits/rejected": 1.6542516946792603,
+      "logps/chosen": -9.177361488342285,
+      "logps/rejected": -10.368098258972168,
+      "loss": 0.4786,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.177361488342285,
+      "rewards/margins": 1.1907374858856201,
+      "rewards/rejected": -10.368098258972168,
+      "semantic_entropy": 0.00208297255448997,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 23.75794088245115,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": 1.4865071773529053,
+      "logits/rejected": 1.5895024538040161,
+      "logps/chosen": -9.169473648071289,
+      "logps/rejected": -10.039076805114746,
+      "loss": 0.5098,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.169473648071289,
+      "rewards/margins": 0.8696044683456421,
+      "rewards/rejected": -10.039076805114746,
+      "semantic_entropy": 0.001707130460999906,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 18.143921236764704,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": 1.588544249534607,
+      "logits/rejected": 1.6502994298934937,
+      "logps/chosen": -9.187943458557129,
+      "logps/rejected": -10.361930847167969,
+      "loss": 0.4534,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.187943458557129,
+      "rewards/margins": 1.173987865447998,
+      "rewards/rejected": -10.361930847167969,
+      "semantic_entropy": 0.0019446357619017363,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 19.009312818648585,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": 1.4991474151611328,
+      "logits/rejected": 1.578648567199707,
+      "logps/chosen": -9.08928108215332,
+      "logps/rejected": -10.218600273132324,
+      "loss": 0.5007,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.08928108215332,
+      "rewards/margins": 1.129319667816162,
+      "rewards/rejected": -10.218600273132324,
+      "semantic_entropy": 0.002233815146610141,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 17.482830760967463,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": 1.5356338024139404,
+      "logits/rejected": 1.5995526313781738,
+      "logps/chosen": -8.733567237854004,
+      "logps/rejected": -9.75208854675293,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.733567237854004,
+      "rewards/margins": 1.0185211896896362,
+      "rewards/rejected": -9.75208854675293,
+      "semantic_entropy": 0.002812595572322607,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 29.50789749732162,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": 1.4784553050994873,
+      "logits/rejected": 1.5580496788024902,
+      "logps/chosen": -8.714761734008789,
+      "logps/rejected": -9.708746910095215,
+      "loss": 0.5293,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.714761734008789,
+      "rewards/margins": 0.9939858317375183,
+      "rewards/rejected": -9.708746910095215,
+      "semantic_entropy": 0.002518023131415248,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 21.386365104238504,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": 1.4471932649612427,
+      "logits/rejected": 1.5621120929718018,
+      "logps/chosen": -8.527729988098145,
+      "logps/rejected": -9.543551445007324,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.527729988098145,
+      "rewards/margins": 1.0158214569091797,
+      "rewards/rejected": -9.543551445007324,
+      "semantic_entropy": 0.003427696181461215,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 25.70625032866536,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": 1.4503463506698608,
+      "logits/rejected": 1.538444995880127,
+      "logps/chosen": -8.535593032836914,
+      "logps/rejected": -9.541770935058594,
+      "loss": 0.5209,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.535593032836914,
+      "rewards/margins": 1.0061774253845215,
+      "rewards/rejected": -9.541770935058594,
+      "semantic_entropy": 0.0032459083013236523,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 24.4150873655686,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": 1.3809781074523926,
+      "logits/rejected": 1.475890040397644,
+      "logps/chosen": -8.557419776916504,
+      "logps/rejected": -9.487534523010254,
+      "loss": 0.5056,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.557419776916504,
+      "rewards/margins": 0.9301142692565918,
+      "rewards/rejected": -9.487534523010254,
+      "semantic_entropy": 0.00316910189576447,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 25.303291673543914,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": 1.4794120788574219,
+      "logits/rejected": 1.5816013813018799,
+      "logps/chosen": -8.555150985717773,
+      "logps/rejected": -9.49781608581543,
+      "loss": 0.5099,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -8.555150985717773,
+      "rewards/margins": 0.942663311958313,
+      "rewards/rejected": -9.49781608581543,
+      "semantic_entropy": 0.003110460937023163,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 19.262136597977563,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": 1.446498155593872,
+      "logits/rejected": 1.597130298614502,
+      "logps/chosen": -9.022222518920898,
+      "logps/rejected": -9.95930004119873,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.022222518920898,
+      "rewards/margins": 0.9370768666267395,
+      "rewards/rejected": -9.95930004119873,
+      "semantic_entropy": 0.002443476812914014,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 28.01118128170372,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": 1.5541894435882568,
+      "logits/rejected": 1.6085865497589111,
+      "logps/chosen": -9.058477401733398,
+      "logps/rejected": -10.006900787353516,
+      "loss": 0.523,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.058477401733398,
+      "rewards/margins": 0.9484248161315918,
+      "rewards/rejected": -10.006900787353516,
+      "semantic_entropy": 0.002096354030072689,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 16.400618383721774,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": 1.5672190189361572,
+      "logits/rejected": 1.6483913660049438,
+      "logps/chosen": -9.431915283203125,
+      "logps/rejected": -10.511835098266602,
+      "loss": 0.4485,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.431915283203125,
+      "rewards/margins": 1.0799185037612915,
+      "rewards/rejected": -10.511835098266602,
+      "semantic_entropy": 0.0014761926140636206,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 21.727658206963017,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": 1.4974210262298584,
+      "logits/rejected": 1.5700048208236694,
+      "logps/chosen": -9.515573501586914,
+      "logps/rejected": -10.426743507385254,
+      "loss": 0.5087,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.515573501586914,
+      "rewards/margins": 0.9111686944961548,
+      "rewards/rejected": -10.426743507385254,
+      "semantic_entropy": 0.0013459009351208806,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 26.104044631850556,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": 1.5503462553024292,
+      "logits/rejected": 1.5834391117095947,
+      "logps/chosen": -9.445427894592285,
+      "logps/rejected": -10.280699729919434,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.445427894592285,
+      "rewards/margins": 0.8352718353271484,
+      "rewards/rejected": -10.280699729919434,
+      "semantic_entropy": 0.0015468759229406714,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 16.84707384898157,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": 1.4859682321548462,
+      "logits/rejected": 1.5847077369689941,
+      "logps/chosen": -9.432461738586426,
+      "logps/rejected": -10.591154098510742,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.432461738586426,
+      "rewards/margins": 1.1586925983428955,
+      "rewards/rejected": -10.591154098510742,
+      "semantic_entropy": 0.001635840511880815,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 19.84132799372696,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": 1.5477855205535889,
+      "logits/rejected": 1.641923189163208,
+      "logps/chosen": -9.205756187438965,
+      "logps/rejected": -10.493422508239746,
+      "loss": 0.4631,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.205756187438965,
+      "rewards/margins": 1.2876663208007812,
+      "rewards/rejected": -10.493422508239746,
+      "semantic_entropy": 0.0017871132586151361,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 23.33580283042589,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": 1.4906589984893799,
+      "logits/rejected": 1.5234686136245728,
+      "logps/chosen": -9.18577766418457,
+      "logps/rejected": -10.161766052246094,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.18577766418457,
+      "rewards/margins": 0.9759882688522339,
+      "rewards/rejected": -10.161766052246094,
+      "semantic_entropy": 0.002067455556243658,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 21.216241651862635,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": 1.5446043014526367,
+      "logits/rejected": 1.5702136754989624,
+      "logps/chosen": -9.186358451843262,
+      "logps/rejected": -10.076950073242188,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.186358451843262,
+      "rewards/margins": 0.8905915021896362,
+      "rewards/rejected": -10.076950073242188,
+      "semantic_entropy": 0.002130972221493721,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 24.418325870174755,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": 1.4704771041870117,
+      "logits/rejected": 1.5745267868041992,
+      "logps/chosen": -9.060630798339844,
+      "logps/rejected": -9.861671447753906,
+      "loss": 0.522,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.060630798339844,
+      "rewards/margins": 0.8010417222976685,
+      "rewards/rejected": -9.861671447753906,
+      "semantic_entropy": 0.0024001654237508774,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 21.280195071325622,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": 1.473073124885559,
+      "logits/rejected": 1.5634665489196777,
+      "logps/chosen": -8.961590766906738,
+      "logps/rejected": -10.119363784790039,
+      "loss": 0.4704,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.961590766906738,
+      "rewards/margins": 1.1577739715576172,
+      "rewards/rejected": -10.119363784790039,
+      "semantic_entropy": 0.002532877027988434,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 21.8418038500209,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": 1.4099845886230469,
+      "logits/rejected": 1.5033830404281616,
+      "logps/chosen": -8.871881484985352,
+      "logps/rejected": -10.012157440185547,
+      "loss": 0.4474,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.871881484985352,
+      "rewards/margins": 1.1402757167816162,
+      "rewards/rejected": -10.012157440185547,
+      "semantic_entropy": 0.0023427356500178576,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 22.672483134577046,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": 1.4795199632644653,
+      "logits/rejected": 1.5488849878311157,
+      "logps/chosen": -8.952245712280273,
+      "logps/rejected": -10.014863967895508,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.952245712280273,
+      "rewards/margins": 1.0626198053359985,
+      "rewards/rejected": -10.014863967895508,
+      "semantic_entropy": 0.002382822334766388,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 18.445862513812127,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": 1.382584810256958,
+      "logits/rejected": 1.4704158306121826,
+      "logps/chosen": -8.761442184448242,
+      "logps/rejected": -9.80408763885498,
+      "loss": 0.4602,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.761442184448242,
+      "rewards/margins": 1.0426462888717651,
+      "rewards/rejected": -9.80408763885498,
+      "semantic_entropy": 0.0027466327883303165,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 21.91010020201385,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": 1.4342511892318726,
+      "logits/rejected": 1.5007261037826538,
+      "logps/chosen": -8.898605346679688,
+      "logps/rejected": -9.936822891235352,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.898605346679688,
+      "rewards/margins": 1.0382182598114014,
+      "rewards/rejected": -9.936822891235352,
+      "semantic_entropy": 0.00258980062790215,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 31.784053134915364,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": 1.456131935119629,
+      "logits/rejected": 1.5059250593185425,
+      "logps/chosen": -9.211524963378906,
+      "logps/rejected": -10.083944320678711,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.211524963378906,
+      "rewards/margins": 0.8724196553230286,
+      "rewards/rejected": -10.083944320678711,
+      "semantic_entropy": 0.0018724482506513596,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 26.520919336244123,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": 1.4616672992706299,
+      "logits/rejected": 1.5543665885925293,
+      "logps/chosen": -9.438743591308594,
+      "logps/rejected": -10.334070205688477,
+      "loss": 0.5362,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.438743591308594,
+      "rewards/margins": 0.8953263163566589,
+      "rewards/rejected": -10.334070205688477,
+      "semantic_entropy": 0.0015787252923473716,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 27.2870089461331,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": 1.4702866077423096,
+      "logits/rejected": 1.5846493244171143,
+      "logps/chosen": -9.516846656799316,
+      "logps/rejected": -10.432823181152344,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.516846656799316,
+      "rewards/margins": 0.9159752726554871,
+      "rewards/rejected": -10.432823181152344,
+      "semantic_entropy": 0.0013532252050936222,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 21.949305810710594,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": 1.444471836090088,
+      "logits/rejected": 1.5971415042877197,
+      "logps/chosen": -9.346989631652832,
+      "logps/rejected": -10.579014778137207,
+      "loss": 0.4377,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.346989631652832,
+      "rewards/margins": 1.2320261001586914,
+      "rewards/rejected": -10.579014778137207,
+      "semantic_entropy": 0.0018633244326338172,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 20.95258547581731,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": 1.4461894035339355,
+      "logits/rejected": 1.5582057237625122,
+      "logps/chosen": -9.405795097351074,
+      "logps/rejected": -10.212226867675781,
+      "loss": 0.537,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.405795097351074,
+      "rewards/margins": 0.8064318895339966,
+      "rewards/rejected": -10.212226867675781,
+      "semantic_entropy": 0.0017630744259804487,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 24.657144950052515,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": 1.4557251930236816,
+      "logits/rejected": 1.5350269079208374,
+      "logps/chosen": -9.127802848815918,
+      "logps/rejected": -10.263969421386719,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.127802848815918,
+      "rewards/margins": 1.1361669301986694,
+      "rewards/rejected": -10.263969421386719,
+      "semantic_entropy": 0.0021912273950874805,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 22.162655716798326,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": 1.3766008615493774,
+      "logits/rejected": 1.4575681686401367,
+      "logps/chosen": -8.819391250610352,
+      "logps/rejected": -9.700312614440918,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.819391250610352,
+      "rewards/margins": 0.8809207677841187,
+      "rewards/rejected": -9.700312614440918,
+      "semantic_entropy": 0.002760782837867737,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 29.399991177675577,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": 1.3292288780212402,
+      "logits/rejected": 1.4387314319610596,
+      "logps/chosen": -8.709214210510254,
+      "logps/rejected": -9.862582206726074,
+      "loss": 0.451,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.709214210510254,
+      "rewards/margins": 1.1533674001693726,
+      "rewards/rejected": -9.862582206726074,
+      "semantic_entropy": 0.003011134685948491,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 18.653352323369948,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": 1.369612455368042,
+      "logits/rejected": 1.4335711002349854,
+      "logps/chosen": -8.789046287536621,
+      "logps/rejected": -9.806333541870117,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.789046287536621,
+      "rewards/margins": 1.017287015914917,
+      "rewards/rejected": -9.806333541870117,
+      "semantic_entropy": 0.002845165552571416,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 22.217260896942022,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": 1.3666447401046753,
+      "logits/rejected": 1.4900730848312378,
+      "logps/chosen": -8.871742248535156,
+      "logps/rejected": -9.85488224029541,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.871742248535156,
+      "rewards/margins": 0.9831396341323853,
+      "rewards/rejected": -9.85488224029541,
+      "semantic_entropy": 0.0023460527881979942,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 23.409117662704528,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": 1.3734443187713623,
+      "logits/rejected": 1.453566074371338,
+      "logps/chosen": -9.178972244262695,
+      "logps/rejected": -9.913165092468262,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -9.178972244262695,
+      "rewards/margins": 0.7341926693916321,
+      "rewards/rejected": -9.913165092468262,
+      "semantic_entropy": 0.002333291107788682,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 20.918477746614087,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": 1.4548704624176025,
+      "logits/rejected": 1.5543843507766724,
+      "logps/chosen": -9.166410446166992,
+      "logps/rejected": -10.090686798095703,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.166410446166992,
+      "rewards/margins": 0.9242767095565796,
+      "rewards/rejected": -10.090686798095703,
+      "semantic_entropy": 0.0022218418307602406,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 26.516844264169315,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": 1.4171142578125,
+      "logits/rejected": 1.5058948993682861,
+      "logps/chosen": -8.995026588439941,
+      "logps/rejected": -9.952535629272461,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.995026588439941,
+      "rewards/margins": 0.9575104713439941,
+      "rewards/rejected": -9.952535629272461,
+      "semantic_entropy": 0.0022883848287165165,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 25.035434437061753,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": 1.3678803443908691,
+      "logits/rejected": 1.4347221851348877,
+      "logps/chosen": -9.04840087890625,
+      "logps/rejected": -9.902055740356445,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.04840087890625,
+      "rewards/margins": 0.8536549806594849,
+      "rewards/rejected": -9.902055740356445,
+      "semantic_entropy": 0.00227540684863925,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 25.33852735218735,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": 1.4378461837768555,
+      "logits/rejected": 1.5412635803222656,
+      "logps/chosen": -8.885682106018066,
+      "logps/rejected": -9.858682632446289,
+      "loss": 0.4877,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.885682106018066,
+      "rewards/margins": 0.973000705242157,
+      "rewards/rejected": -9.858682632446289,
+      "semantic_entropy": 0.0025360703002661467,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 17.386304853905195,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": 1.4217267036437988,
+      "logits/rejected": 1.4756557941436768,
+      "logps/chosen": -8.806241989135742,
+      "logps/rejected": -9.654108047485352,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.806241989135742,
+      "rewards/margins": 0.8478654026985168,
+      "rewards/rejected": -9.654108047485352,
+      "semantic_entropy": 0.0024697603657841682,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 21.550453991458607,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": 1.390870213508606,
+      "logits/rejected": 1.432539939880371,
+      "logps/chosen": -8.635198593139648,
+      "logps/rejected": -9.709808349609375,
+      "loss": 0.464,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.635198593139648,
+      "rewards/margins": 1.0746097564697266,
+      "rewards/rejected": -9.709808349609375,
+      "semantic_entropy": 0.002927431371062994,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 24.90357478492923,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": 1.3805229663848877,
+      "logits/rejected": 1.4759821891784668,
+      "logps/chosen": -8.621747970581055,
+      "logps/rejected": -9.80329704284668,
+      "loss": 0.4732,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.621747970581055,
+      "rewards/margins": 1.1815496683120728,
+      "rewards/rejected": -9.80329704284668,
+      "semantic_entropy": 0.0029972377233207226,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 17.82648149464588,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": 1.4264099597930908,
+      "logits/rejected": 1.4067127704620361,
+      "logps/chosen": -8.737417221069336,
+      "logps/rejected": -9.573492050170898,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.737417221069336,
+      "rewards/margins": 0.8360740542411804,
+      "rewards/rejected": -9.573492050170898,
+      "semantic_entropy": 0.002560330554842949,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 28.55995164795418,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": 1.3373901844024658,
+      "logits/rejected": 1.4552834033966064,
+      "logps/chosen": -8.78300952911377,
+      "logps/rejected": -9.90458869934082,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.78300952911377,
+      "rewards/margins": 1.1215803623199463,
+      "rewards/rejected": -9.90458869934082,
+      "semantic_entropy": 0.002698799129575491,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 23.43284713448351,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": 1.238858938217163,
+      "logits/rejected": 1.364005446434021,
+      "logps/chosen": -8.7960844039917,
+      "logps/rejected": -9.684962272644043,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.7960844039917,
+      "rewards/margins": 0.8888777494430542,
+      "rewards/rejected": -9.684962272644043,
+      "semantic_entropy": 0.0026077846996486187,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 19.834930068650173,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": 1.2806402444839478,
+      "logits/rejected": 1.3462021350860596,
+      "logps/chosen": -9.013236999511719,
+      "logps/rejected": -10.023857116699219,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.013236999511719,
+      "rewards/margins": 1.010617971420288,
+      "rewards/rejected": -10.023857116699219,
+      "semantic_entropy": 0.0020977004896849394,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 26.566918391197426,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": 1.3532203435897827,
+      "logits/rejected": 1.420139193534851,
+      "logps/chosen": -9.195432662963867,
+      "logps/rejected": -9.993632316589355,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.195432662963867,
+      "rewards/margins": 0.7981998324394226,
+      "rewards/rejected": -9.993632316589355,
+      "semantic_entropy": 0.0019540577195584774,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 23.88255352058365,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": 1.334280252456665,
+      "logits/rejected": 1.4078104496002197,
+      "logps/chosen": -8.778676986694336,
+      "logps/rejected": -9.887304306030273,
+      "loss": 0.4513,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.778676986694336,
+      "rewards/margins": 1.1086275577545166,
+      "rewards/rejected": -9.887304306030273,
+      "semantic_entropy": 0.0027104257605969906,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 24.215344599310438,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": 1.3388506174087524,
+      "logits/rejected": 1.4423941373825073,
+      "logps/chosen": -8.978914260864258,
+      "logps/rejected": -9.998015403747559,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.978914260864258,
+      "rewards/margins": 1.0191015005111694,
+      "rewards/rejected": -9.998015403747559,
+      "semantic_entropy": 0.0023236635606735945,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 24.815674232925165,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": 1.3499490022659302,
+      "logits/rejected": 1.4477221965789795,
+      "logps/chosen": -8.803632736206055,
+      "logps/rejected": -9.696292877197266,
+      "loss": 0.501,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.803632736206055,
+      "rewards/margins": 0.8926607966423035,
+      "rewards/rejected": -9.696292877197266,
+      "semantic_entropy": 0.0024066984187811613,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 21.027450348295872,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": 1.3931233882904053,
+      "logits/rejected": 1.4372663497924805,
+      "logps/chosen": -8.543787002563477,
+      "logps/rejected": -9.412200927734375,
+      "loss": 0.551,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -8.543787002563477,
+      "rewards/margins": 0.868414044380188,
+      "rewards/rejected": -9.412200927734375,
+      "semantic_entropy": 0.0030270537827163935,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 19.078804475025134,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": 1.3064452409744263,
+      "logits/rejected": 1.4189800024032593,
+      "logps/chosen": -8.807589530944824,
+      "logps/rejected": -9.77020263671875,
+      "loss": 0.4717,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -8.807589530944824,
+      "rewards/margins": 0.962612509727478,
+      "rewards/rejected": -9.77020263671875,
+      "semantic_entropy": 0.0024364350829273462,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 24.016607540857905,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": 1.3972842693328857,
+      "logits/rejected": 1.4768155813217163,
+      "logps/chosen": -8.881470680236816,
+      "logps/rejected": -9.918227195739746,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.881470680236816,
+      "rewards/margins": 1.0367575883865356,
+      "rewards/rejected": -9.918227195739746,
+      "semantic_entropy": 0.00256557809188962,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 38.34963635227672,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": 1.407387137413025,
+      "logits/rejected": 1.5070137977600098,
+      "logps/chosen": -8.959281921386719,
+      "logps/rejected": -9.95294189453125,
+      "loss": 0.5161,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.959281921386719,
+      "rewards/margins": 0.993659496307373,
+      "rewards/rejected": -9.95294189453125,
+      "semantic_entropy": 0.0029083569534122944,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 30.824650764551436,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": 1.3412138223648071,
+      "logits/rejected": 1.4596610069274902,
+      "logps/chosen": -8.714391708374023,
+      "logps/rejected": -9.62089729309082,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.714391708374023,
+      "rewards/margins": 0.9065059423446655,
+      "rewards/rejected": -9.62089729309082,
+      "semantic_entropy": 0.00292821298353374,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 20.852237398556845,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": 1.3490220308303833,
+      "logits/rejected": 1.5013482570648193,
+      "logps/chosen": -8.605868339538574,
+      "logps/rejected": -9.815937042236328,
+      "loss": 0.4273,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.605868339538574,
+      "rewards/margins": 1.2100684642791748,
+      "rewards/rejected": -9.815937042236328,
+      "semantic_entropy": 0.002925614593550563,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 34.40540777799309,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": 1.3056410551071167,
+      "logits/rejected": 1.4626327753067017,
+      "logps/chosen": -8.616663932800293,
+      "logps/rejected": -9.704048156738281,
+      "loss": 0.5023,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.616663932800293,
+      "rewards/margins": 1.0873849391937256,
+      "rewards/rejected": -9.704048156738281,
+      "semantic_entropy": 0.0034379579592496157,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 22.96404387222335,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": 1.426653265953064,
+      "logits/rejected": 1.4489518404006958,
+      "logps/chosen": -8.686907768249512,
+      "logps/rejected": -9.805135726928711,
+      "loss": 0.4667,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.686907768249512,
+      "rewards/margins": 1.1182278394699097,
+      "rewards/rejected": -9.805135726928711,
+      "semantic_entropy": 0.002787970704957843,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 18.351222067377243,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": 1.4044121503829956,
+      "logits/rejected": 1.529573678970337,
+      "logps/chosen": -9.031078338623047,
+      "logps/rejected": -10.060023307800293,
+      "loss": 0.4694,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.031078338623047,
+      "rewards/margins": 1.028944730758667,
+      "rewards/rejected": -10.060023307800293,
+      "semantic_entropy": 0.002074579708278179,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 22.5373305804796,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": 1.4100029468536377,
+      "logits/rejected": 1.5267999172210693,
+      "logps/chosen": -9.287673950195312,
+      "logps/rejected": -10.477519035339355,
+      "loss": 0.4568,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.287673950195312,
+      "rewards/margins": 1.1898446083068848,
+      "rewards/rejected": -10.477519035339355,
+      "semantic_entropy": 0.0016608504811301827,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 19.23312901806617,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": 1.457642912864685,
+      "logits/rejected": 1.4848308563232422,
+      "logps/chosen": -9.176215171813965,
+      "logps/rejected": -10.324426651000977,
+      "loss": 0.454,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.176215171813965,
+      "rewards/margins": 1.1482112407684326,
+      "rewards/rejected": -10.324426651000977,
+      "semantic_entropy": 0.0022915401495993137,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 19.350939928732608,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": 1.3565762042999268,
+      "logits/rejected": 1.436859369277954,
+      "logps/chosen": -9.130460739135742,
+      "logps/rejected": -10.147039413452148,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.130460739135742,
+      "rewards/margins": 1.0165780782699585,
+      "rewards/rejected": -10.147039413452148,
+      "semantic_entropy": 0.00199532276019454,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 17.918752319301145,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": 1.3714624643325806,
+      "logits/rejected": 1.4519484043121338,
+      "logps/chosen": -9.17676830291748,
+      "logps/rejected": -10.011459350585938,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.17676830291748,
+      "rewards/margins": 0.8346899747848511,
+      "rewards/rejected": -10.011459350585938,
+      "semantic_entropy": 0.0018281477969139814,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 30.08960482046981,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": 1.3898823261260986,
+      "logits/rejected": 1.4446918964385986,
+      "logps/chosen": -9.117020606994629,
+      "logps/rejected": -10.000941276550293,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.117020606994629,
+      "rewards/margins": 0.8839200139045715,
+      "rewards/rejected": -10.000941276550293,
+      "semantic_entropy": 0.0019779205322265625,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 24.00223143232026,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": 1.3938207626342773,
+      "logits/rejected": 1.4990066289901733,
+      "logps/chosen": -8.891989707946777,
+      "logps/rejected": -9.914690971374512,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.891989707946777,
+      "rewards/margins": 1.0227010250091553,
+      "rewards/rejected": -9.914690971374512,
+      "semantic_entropy": 0.0023722308687865734,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 17.613182150840984,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": 1.391399621963501,
+      "logits/rejected": 1.5063310861587524,
+      "logps/chosen": -8.578347206115723,
+      "logps/rejected": -9.771947860717773,
+      "loss": 0.4267,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.578347206115723,
+      "rewards/margins": 1.1936001777648926,
+      "rewards/rejected": -9.771947860717773,
+      "semantic_entropy": 0.0032676062546670437,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 17.496887986667375,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": 1.360595464706421,
+      "logits/rejected": 1.4230304956436157,
+      "logps/chosen": -8.729754447937012,
+      "logps/rejected": -9.858755111694336,
+      "loss": 0.4596,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.729754447937012,
+      "rewards/margins": 1.1290003061294556,
+      "rewards/rejected": -9.858755111694336,
+      "semantic_entropy": 0.003013763576745987,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 22.40900419939581,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": 1.3398988246917725,
+      "logits/rejected": 1.4405022859573364,
+      "logps/chosen": -8.662479400634766,
+      "logps/rejected": -9.675493240356445,
+      "loss": 0.481,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.662479400634766,
+      "rewards/margins": 1.0130139589309692,
+      "rewards/rejected": -9.675493240356445,
+      "semantic_entropy": 0.002807116601616144,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 16.945164046693417,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": 1.3781074285507202,
+      "logits/rejected": 1.4572255611419678,
+      "logps/chosen": -8.682049751281738,
+      "logps/rejected": -9.892243385314941,
+      "loss": 0.4322,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.682049751281738,
+      "rewards/margins": 1.21019446849823,
+      "rewards/rejected": -9.892243385314941,
+      "semantic_entropy": 0.0029278115835040808,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 25.429810586801697,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": 1.4200975894927979,
+      "logits/rejected": 1.4886587858200073,
+      "logps/chosen": -8.73933219909668,
+      "logps/rejected": -9.8605375289917,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.73933219909668,
+      "rewards/margins": 1.121203064918518,
+      "rewards/rejected": -9.8605375289917,
+      "semantic_entropy": 0.002721203491091728,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 21.846154417725824,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": 1.3389413356781006,
+      "logits/rejected": 1.4276649951934814,
+      "logps/chosen": -8.829744338989258,
+      "logps/rejected": -9.880697250366211,
+      "loss": 0.4837,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.829744338989258,
+      "rewards/margins": 1.0509527921676636,
+      "rewards/rejected": -9.880697250366211,
+      "semantic_entropy": 0.0027480546850711107,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 21.30917604463369,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": 1.4596788883209229,
+      "logits/rejected": 1.534031629562378,
+      "logps/chosen": -8.962684631347656,
+      "logps/rejected": -9.927931785583496,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.962684631347656,
+      "rewards/margins": 0.965247631072998,
+      "rewards/rejected": -9.927931785583496,
+      "semantic_entropy": 0.002392445458099246,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 14.434796029324703,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": 1.362815499305725,
+      "logits/rejected": 1.4356257915496826,
+      "logps/chosen": -8.6842622756958,
+      "logps/rejected": -9.785943984985352,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.6842622756958,
+      "rewards/margins": 1.1016814708709717,
+      "rewards/rejected": -9.785943984985352,
+      "semantic_entropy": 0.003006249200552702,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 23.806179051231396,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": 1.4174171686172485,
+      "logits/rejected": 1.4843686819076538,
+      "logps/chosen": -8.935274124145508,
+      "logps/rejected": -9.890968322753906,
+      "loss": 0.5178,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -8.935274124145508,
+      "rewards/margins": 0.9556947946548462,
+      "rewards/rejected": -9.890968322753906,
+      "semantic_entropy": 0.0025258921086788177,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 18.648630251455785,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": 1.3888474702835083,
+      "logits/rejected": 1.4548256397247314,
+      "logps/chosen": -8.759489059448242,
+      "logps/rejected": -9.653459548950195,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.759489059448242,
+      "rewards/margins": 0.8939695358276367,
+      "rewards/rejected": -9.653459548950195,
+      "semantic_entropy": 0.002887281123548746,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 22.73840363617096,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": 1.3133025169372559,
+      "logits/rejected": 1.4226750135421753,
+      "logps/chosen": -8.789426803588867,
+      "logps/rejected": -9.88960075378418,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.789426803588867,
+      "rewards/margins": 1.1001733541488647,
+      "rewards/rejected": -9.88960075378418,
+      "semantic_entropy": 0.0032505816780030727,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 30.76862276215726,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": 1.3512465953826904,
+      "logits/rejected": 1.3566533327102661,
+      "logps/chosen": -8.438799858093262,
+      "logps/rejected": -9.353658676147461,
+      "loss": 0.5252,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.438799858093262,
+      "rewards/margins": 0.9148581624031067,
+      "rewards/rejected": -9.353658676147461,
+      "semantic_entropy": 0.0037102126516401768,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 16.414218784959694,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": 1.3738600015640259,
+      "logits/rejected": 1.3996831178665161,
+      "logps/chosen": -8.360509872436523,
+      "logps/rejected": -9.395818710327148,
+      "loss": 0.502,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.360509872436523,
+      "rewards/margins": 1.0353081226348877,
+      "rewards/rejected": -9.395818710327148,
+      "semantic_entropy": 0.004059081431478262,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 25.849810249978756,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": 1.3306156396865845,
+      "logits/rejected": 1.3967326879501343,
+      "logps/chosen": -8.509716987609863,
+      "logps/rejected": -9.52521800994873,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.509716987609863,
+      "rewards/margins": 1.0155019760131836,
+      "rewards/rejected": -9.52521800994873,
+      "semantic_entropy": 0.003359501715749502,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": 1.5073561668395996,
+      "eval_logits/rejected": 1.5667141675949097,
+      "eval_logps/chosen": -8.781377792358398,
+      "eval_logps/rejected": -9.710713386535645,
+      "eval_loss": 0.5317959189414978,
+      "eval_rewards/accuracies": 0.7255192995071411,
+      "eval_rewards/chosen": -8.781377792358398,
+      "eval_rewards/margins": 0.9293367862701416,
+      "eval_rewards/rejected": -9.710713386535645,
+      "eval_runtime": 34.6038,
+      "eval_samples_per_second": 38.869,
+      "eval_semantic_entropy": 0.0028174184262752533,
+      "eval_steps_per_second": 9.739,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 19.074900038590364,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": 1.3281657695770264,
+      "logits/rejected": 1.4117894172668457,
+      "logps/chosen": -8.730180740356445,
+      "logps/rejected": -9.6874418258667,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.730180740356445,
+      "rewards/margins": 0.9572622179985046,
+      "rewards/rejected": -9.6874418258667,
+      "semantic_entropy": 0.003087426070123911,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 23.930017615418897,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": 1.4048469066619873,
+      "logits/rejected": 1.462122917175293,
+      "logps/chosen": -8.789390563964844,
+      "logps/rejected": -9.749144554138184,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.789390563964844,
+      "rewards/margins": 0.9597553014755249,
+      "rewards/rejected": -9.749144554138184,
+      "semantic_entropy": 0.0028791739605367184,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 15.270691780632738,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": 1.3611500263214111,
+      "logits/rejected": 1.4282476902008057,
+      "logps/chosen": -8.856307029724121,
+      "logps/rejected": -10.035266876220703,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -8.856307029724121,
+      "rewards/margins": 1.178959846496582,
+      "rewards/rejected": -10.035266876220703,
+      "semantic_entropy": 0.002467815298587084,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 20.40403425710895,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": 1.3527324199676514,
+      "logits/rejected": 1.458662509918213,
+      "logps/chosen": -8.956108093261719,
+      "logps/rejected": -9.771733283996582,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.956108093261719,
+      "rewards/margins": 0.8156250715255737,
+      "rewards/rejected": -9.771733283996582,
+      "semantic_entropy": 0.0025175369810312986,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 18.78839187495872,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": 1.3605597019195557,
+      "logits/rejected": 1.4761570692062378,
+      "logps/chosen": -8.866493225097656,
+      "logps/rejected": -9.913625717163086,
+      "loss": 0.4696,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.866493225097656,
+      "rewards/margins": 1.047132134437561,
+      "rewards/rejected": -9.913625717163086,
+      "semantic_entropy": 0.002443327335640788,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 15.111039453596693,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": 1.3532849550247192,
+      "logits/rejected": 1.447892427444458,
+      "logps/chosen": -8.536003112792969,
+      "logps/rejected": -9.617258071899414,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.536003112792969,
+      "rewards/margins": 1.0812547206878662,
+      "rewards/rejected": -9.617258071899414,
+      "semantic_entropy": 0.0031031756661832333,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 20.19154872530171,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": 1.4047647714614868,
+      "logits/rejected": 1.501410961151123,
+      "logps/chosen": -8.890284538269043,
+      "logps/rejected": -10.056889533996582,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -8.890284538269043,
+      "rewards/margins": 1.1666046380996704,
+      "rewards/rejected": -10.056889533996582,
+      "semantic_entropy": 0.002500087721273303,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 20.900578617111748,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": 1.4015910625457764,
+      "logits/rejected": 1.4419715404510498,
+      "logps/chosen": -8.744017601013184,
+      "logps/rejected": -9.691323280334473,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.744017601013184,
+      "rewards/margins": 0.9473053812980652,
+      "rewards/rejected": -9.691323280334473,
+      "semantic_entropy": 0.002974584000185132,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 18.401808714357863,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": 1.4739038944244385,
+      "logits/rejected": 1.5210344791412354,
+      "logps/chosen": -8.868200302124023,
+      "logps/rejected": -9.828581809997559,
+      "loss": 0.5621,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -8.868200302124023,
+      "rewards/margins": 0.9603812098503113,
+      "rewards/rejected": -9.828581809997559,
+      "semantic_entropy": 0.0028297235257923603,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 17.334663099926633,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": 1.3810946941375732,
+      "logits/rejected": 1.4490197896957397,
+      "logps/chosen": -8.932979583740234,
+      "logps/rejected": -10.094772338867188,
+      "loss": 0.461,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.932979583740234,
+      "rewards/margins": 1.1617931127548218,
+      "rewards/rejected": -10.094772338867188,
+      "semantic_entropy": 0.002513275947421789,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 19.42769773097853,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": 1.3801032304763794,
+      "logits/rejected": 1.486676573753357,
+      "logps/chosen": -8.910977363586426,
+      "logps/rejected": -9.867474555969238,
+      "loss": 0.483,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.910977363586426,
+      "rewards/margins": 0.9564973711967468,
+      "rewards/rejected": -9.867474555969238,
+      "semantic_entropy": 0.0026378934271633625,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 19.030777761214626,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": 1.3764504194259644,
+      "logits/rejected": 1.492344617843628,
+      "logps/chosen": -9.025213241577148,
+      "logps/rejected": -9.9039945602417,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.025213241577148,
+      "rewards/margins": 0.878780722618103,
+      "rewards/rejected": -9.9039945602417,
+      "semantic_entropy": 0.0020701594185084105,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 20.185929506946096,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": 1.419599175453186,
+      "logits/rejected": 1.504597783088684,
+      "logps/chosen": -8.93053913116455,
+      "logps/rejected": -9.816963195800781,
+      "loss": 0.51,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.93053913116455,
+      "rewards/margins": 0.8864244222640991,
+      "rewards/rejected": -9.816963195800781,
+      "semantic_entropy": 0.002383794402703643,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 21.917356089294145,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": 1.4868673086166382,
+      "logits/rejected": 1.570197582244873,
+      "logps/chosen": -8.999963760375977,
+      "logps/rejected": -9.938414573669434,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.999963760375977,
+      "rewards/margins": 0.9384516477584839,
+      "rewards/rejected": -9.938414573669434,
+      "semantic_entropy": 0.002094829920679331,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 22.719172346713275,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": 1.376840591430664,
+      "logits/rejected": 1.4644696712493896,
+      "logps/chosen": -8.935644149780273,
+      "logps/rejected": -9.803464889526367,
+      "loss": 0.5276,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.935644149780273,
+      "rewards/margins": 0.8678204417228699,
+      "rewards/rejected": -9.803464889526367,
+      "semantic_entropy": 0.002422671765089035,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 19.129004275349434,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": 1.446630835533142,
+      "logits/rejected": 1.5216937065124512,
+      "logps/chosen": -8.913363456726074,
+      "logps/rejected": -9.97227954864502,
+      "loss": 0.467,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -8.913363456726074,
+      "rewards/margins": 1.0589138269424438,
+      "rewards/rejected": -9.97227954864502,
+      "semantic_entropy": 0.0024961228482425213,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 16.96669693342859,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": 1.4310683012008667,
+      "logits/rejected": 1.5208488702774048,
+      "logps/chosen": -8.971841812133789,
+      "logps/rejected": -10.059237480163574,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.971841812133789,
+      "rewards/margins": 1.0873949527740479,
+      "rewards/rejected": -10.059237480163574,
+      "semantic_entropy": 0.0021926448680460453,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 17.47426955138407,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": 1.4393575191497803,
+      "logits/rejected": 1.5399881601333618,
+      "logps/chosen": -9.123726844787598,
+      "logps/rejected": -10.197049140930176,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.123726844787598,
+      "rewards/margins": 1.0733228921890259,
+      "rewards/rejected": -10.197049140930176,
+      "semantic_entropy": 0.0020001751836389303,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 20.225881627530626,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": 1.4741970300674438,
+      "logits/rejected": 1.5637295246124268,
+      "logps/chosen": -9.10370922088623,
+      "logps/rejected": -10.148512840270996,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.10370922088623,
+      "rewards/margins": 1.0448031425476074,
+      "rewards/rejected": -10.148512840270996,
+      "semantic_entropy": 0.002272148383781314,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 18.47117255374998,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": 1.4469518661499023,
+      "logits/rejected": 1.5238556861877441,
+      "logps/chosen": -9.073151588439941,
+      "logps/rejected": -10.227090835571289,
+      "loss": 0.4434,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.073151588439941,
+      "rewards/margins": 1.1539392471313477,
+      "rewards/rejected": -10.227090835571289,
+      "semantic_entropy": 0.0018197160679847002,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 24.511152623684787,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": 1.5241786241531372,
+      "logits/rejected": 1.5616201162338257,
+      "logps/chosen": -9.052220344543457,
+      "logps/rejected": -10.1766357421875,
+      "loss": 0.4593,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.052220344543457,
+      "rewards/margins": 1.1244161128997803,
+      "rewards/rejected": -10.1766357421875,
+      "semantic_entropy": 0.002216730499640107,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 24.386365953854444,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": 1.4507062435150146,
+      "logits/rejected": 1.5118112564086914,
+      "logps/chosen": -9.227087020874023,
+      "logps/rejected": -10.175787925720215,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.227087020874023,
+      "rewards/margins": 0.9487009048461914,
+      "rewards/rejected": -10.175787925720215,
+      "semantic_entropy": 0.0018378717359155416,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 27.54509130714091,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": 1.43242609500885,
+      "logits/rejected": 1.530622124671936,
+      "logps/chosen": -9.200937271118164,
+      "logps/rejected": -10.38686752319336,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.200937271118164,
+      "rewards/margins": 1.1859301328659058,
+      "rewards/rejected": -10.38686752319336,
+      "semantic_entropy": 0.0019035383593291044,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 20.942453107732405,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": 1.4240140914916992,
+      "logits/rejected": 1.4918062686920166,
+      "logps/chosen": -9.117125511169434,
+      "logps/rejected": -10.162875175476074,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.117125511169434,
+      "rewards/margins": 1.0457494258880615,
+      "rewards/rejected": -10.162875175476074,
+      "semantic_entropy": 0.0019554845057427883,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 27.30472966273389,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": 1.4881393909454346,
+      "logits/rejected": 1.5379546880722046,
+      "logps/chosen": -9.041446685791016,
+      "logps/rejected": -10.001977920532227,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.041446685791016,
+      "rewards/margins": 0.9605310559272766,
+      "rewards/rejected": -10.001977920532227,
+      "semantic_entropy": 0.002475083339959383,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 18.364192536602022,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": 1.318584680557251,
+      "logits/rejected": 1.4121856689453125,
+      "logps/chosen": -8.920633316040039,
+      "logps/rejected": -10.328185081481934,
+      "loss": 0.4413,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -8.920633316040039,
+      "rewards/margins": 1.4075521230697632,
+      "rewards/rejected": -10.328185081481934,
+      "semantic_entropy": 0.0022802776657044888,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 18.45806108110059,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": 1.3468916416168213,
+      "logits/rejected": 1.402174711227417,
+      "logps/chosen": -9.079140663146973,
+      "logps/rejected": -10.04302978515625,
+      "loss": 0.5029,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.079140663146973,
+      "rewards/margins": 0.9638885259628296,
+      "rewards/rejected": -10.04302978515625,
+      "semantic_entropy": 0.0021296185441315174,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 19.52992873815885,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": 1.347177505493164,
+      "logits/rejected": 1.4360207319259644,
+      "logps/chosen": -9.183477401733398,
+      "logps/rejected": -10.404787063598633,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.183477401733398,
+      "rewards/margins": 1.221308708190918,
+      "rewards/rejected": -10.404787063598633,
+      "semantic_entropy": 0.0023944589775055647,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 19.013174632811925,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": 1.376796007156372,
+      "logits/rejected": 1.4541070461273193,
+      "logps/chosen": -9.031773567199707,
+      "logps/rejected": -10.111733436584473,
+      "loss": 0.5005,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.031773567199707,
+      "rewards/margins": 1.0799596309661865,
+      "rewards/rejected": -10.111733436584473,
+      "semantic_entropy": 0.002499754074960947,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 28.624834750868843,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": 1.4194419384002686,
+      "logits/rejected": 1.5414321422576904,
+      "logps/chosen": -9.009321212768555,
+      "logps/rejected": -10.180513381958008,
+      "loss": 0.4235,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.009321212768555,
+      "rewards/margins": 1.1711915731430054,
+      "rewards/rejected": -10.180513381958008,
+      "semantic_entropy": 0.002233207691460848,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 24.11320007432748,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": 1.5115993022918701,
+      "logits/rejected": 1.5894025564193726,
+      "logps/chosen": -9.002803802490234,
+      "logps/rejected": -9.924473762512207,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -9.002803802490234,
+      "rewards/margins": 0.9216697812080383,
+      "rewards/rejected": -9.924473762512207,
+      "semantic_entropy": 0.002451828448101878,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 18.700958678368707,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": 1.3215209245681763,
+      "logits/rejected": 1.4025137424468994,
+      "logps/chosen": -8.853266716003418,
+      "logps/rejected": -9.972722053527832,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -8.853266716003418,
+      "rewards/margins": 1.119455099105835,
+      "rewards/rejected": -9.972722053527832,
+      "semantic_entropy": 0.002616308396682143,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 18.8766048849701,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": 1.391259789466858,
+      "logits/rejected": 1.4566327333450317,
+      "logps/chosen": -8.97907829284668,
+      "logps/rejected": -10.141624450683594,
+      "loss": 0.4703,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.97907829284668,
+      "rewards/margins": 1.1625462770462036,
+      "rewards/rejected": -10.141624450683594,
+      "semantic_entropy": 0.002474232343956828,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 24.836798897283007,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": 1.3666449785232544,
+      "logits/rejected": 1.4855159521102905,
+      "logps/chosen": -9.012104988098145,
+      "logps/rejected": -9.971700668334961,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.012104988098145,
+      "rewards/margins": 0.9595950841903687,
+      "rewards/rejected": -9.971700668334961,
+      "semantic_entropy": 0.002232321770861745,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 31.74553058225806,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": 1.372730016708374,
+      "logits/rejected": 1.4541945457458496,
+      "logps/chosen": -8.831666946411133,
+      "logps/rejected": -9.705374717712402,
+      "loss": 0.534,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -8.831666946411133,
+      "rewards/margins": 0.8737092018127441,
+      "rewards/rejected": -9.705374717712402,
+      "semantic_entropy": 0.002491941209882498,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 16.610061587643095,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": 1.3365932703018188,
+      "logits/rejected": 1.4221614599227905,
+      "logps/chosen": -8.736175537109375,
+      "logps/rejected": -9.710248947143555,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.736175537109375,
+      "rewards/margins": 0.974072277545929,
+      "rewards/rejected": -9.710248947143555,
+      "semantic_entropy": 0.002767590107396245,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 25.30229795532134,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": 1.3805124759674072,
+      "logits/rejected": 1.50020432472229,
+      "logps/chosen": -8.929006576538086,
+      "logps/rejected": -10.11659049987793,
+      "loss": 0.4738,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.929006576538086,
+      "rewards/margins": 1.1875823736190796,
+      "rewards/rejected": -10.11659049987793,
+      "semantic_entropy": 0.0022952775470912457,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 17.680194735414556,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": 1.412115454673767,
+      "logits/rejected": 1.5250195264816284,
+      "logps/chosen": -8.736458778381348,
+      "logps/rejected": -9.927319526672363,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.736458778381348,
+      "rewards/margins": 1.1908600330352783,
+      "rewards/rejected": -9.927319526672363,
+      "semantic_entropy": 0.002859854605048895,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 23.820847800281037,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": 1.4332348108291626,
+      "logits/rejected": 1.5090396404266357,
+      "logps/chosen": -8.903064727783203,
+      "logps/rejected": -10.09050464630127,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.903064727783203,
+      "rewards/margins": 1.187439203262329,
+      "rewards/rejected": -10.09050464630127,
+      "semantic_entropy": 0.002617530059069395,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 14.321217024589917,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": 1.3947592973709106,
+      "logits/rejected": 1.4961799383163452,
+      "logps/chosen": -8.860940933227539,
+      "logps/rejected": -9.822208404541016,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -8.860940933227539,
+      "rewards/margins": 0.9612663388252258,
+      "rewards/rejected": -9.822208404541016,
+      "semantic_entropy": 0.0026545547880232334,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 39.93584870222566,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": 1.3567323684692383,
+      "logits/rejected": 1.4534469842910767,
+      "logps/chosen": -9.02118968963623,
+      "logps/rejected": -9.794608116149902,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.02118968963623,
+      "rewards/margins": 0.7734187245368958,
+      "rewards/rejected": -9.794608116149902,
+      "semantic_entropy": 0.002223126357421279,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 21.83830977643616,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": 1.384690523147583,
+      "logits/rejected": 1.4703580141067505,
+      "logps/chosen": -8.856417655944824,
+      "logps/rejected": -9.79028034210205,
+      "loss": 0.5636,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.856417655944824,
+      "rewards/margins": 0.9338618516921997,
+      "rewards/rejected": -9.79028034210205,
+      "semantic_entropy": 0.002954977797344327,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 22.8745865203803,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": 1.4836395978927612,
+      "logits/rejected": 1.5417969226837158,
+      "logps/chosen": -8.974642753601074,
+      "logps/rejected": -9.995182037353516,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -8.974642753601074,
+      "rewards/margins": 1.0205390453338623,
+      "rewards/rejected": -9.995182037353516,
+      "semantic_entropy": 0.002676154486835003,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 23.994943829804313,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": 1.4433205127716064,
+      "logits/rejected": 1.4914233684539795,
+      "logps/chosen": -9.027420997619629,
+      "logps/rejected": -9.893829345703125,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.027420997619629,
+      "rewards/margins": 0.8664078712463379,
+      "rewards/rejected": -9.893829345703125,
+      "semantic_entropy": 0.002081984654068947,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 20.398459328728247,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": 1.4040393829345703,
+      "logits/rejected": 1.5039653778076172,
+      "logps/chosen": -9.001434326171875,
+      "logps/rejected": -9.943794250488281,
+      "loss": 0.5206,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.001434326171875,
+      "rewards/margins": 0.9423604011535645,
+      "rewards/rejected": -9.943794250488281,
+      "semantic_entropy": 0.0020064213313162327,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 20.43904708907953,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": 1.3954778909683228,
+      "logits/rejected": 1.4594025611877441,
+      "logps/chosen": -9.071332931518555,
+      "logps/rejected": -10.04885482788086,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.071332931518555,
+      "rewards/margins": 0.9775213003158569,
+      "rewards/rejected": -10.04885482788086,
+      "semantic_entropy": 0.002463545184582472,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 19.552628913177028,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": 1.4234349727630615,
+      "logits/rejected": 1.4693269729614258,
+      "logps/chosen": -9.043578147888184,
+      "logps/rejected": -9.974569320678711,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.043578147888184,
+      "rewards/margins": 0.9309908151626587,
+      "rewards/rejected": -9.974569320678711,
+      "semantic_entropy": 0.002301167929545045,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 29.536831481067566,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": 1.4206435680389404,
+      "logits/rejected": 1.4544965028762817,
+      "logps/chosen": -9.031013488769531,
+      "logps/rejected": -10.177419662475586,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.031013488769531,
+      "rewards/margins": 1.146405816078186,
+      "rewards/rejected": -10.177419662475586,
+      "semantic_entropy": 0.0020169729832559824,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 39.84596871022446,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": 1.4386389255523682,
+      "logits/rejected": 1.5043251514434814,
+      "logps/chosen": -9.088945388793945,
+      "logps/rejected": -10.10850715637207,
+      "loss": 0.4875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.088945388793945,
+      "rewards/margins": 1.0195623636245728,
+      "rewards/rejected": -10.10850715637207,
+      "semantic_entropy": 0.001932180835865438,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 15.222207446253247,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": 1.4539210796356201,
+      "logits/rejected": 1.537204384803772,
+      "logps/chosen": -9.401707649230957,
+      "logps/rejected": -10.541633605957031,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.401707649230957,
+      "rewards/margins": 1.1399261951446533,
+      "rewards/rejected": -10.541633605957031,
+      "semantic_entropy": 0.0014313531573861837,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 19.901702505305185,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": 1.4057888984680176,
+      "logits/rejected": 1.4682310819625854,
+      "logps/chosen": -9.663395881652832,
+      "logps/rejected": -10.535517692565918,
+      "loss": 0.52,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.663395881652832,
+      "rewards/margins": 0.872123122215271,
+      "rewards/rejected": -10.535517692565918,
+      "semantic_entropy": 0.0012065216433256865,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 16.90616173059958,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": 1.4263266324996948,
+      "logits/rejected": 1.5227488279342651,
+      "logps/chosen": -9.384716033935547,
+      "logps/rejected": -10.425240516662598,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.384716033935547,
+      "rewards/margins": 1.0405242443084717,
+      "rewards/rejected": -10.425240516662598,
+      "semantic_entropy": 0.0017487381119281054,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 16.60372512731688,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": 1.4605995416641235,
+      "logits/rejected": 1.5014557838439941,
+      "logps/chosen": -9.379908561706543,
+      "logps/rejected": -10.331750869750977,
+      "loss": 0.5556,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.379908561706543,
+      "rewards/margins": 0.9518418312072754,
+      "rewards/rejected": -10.331750869750977,
+      "semantic_entropy": 0.001687846495769918,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 28.27406693183152,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": 1.4176925420761108,
+      "logits/rejected": 1.5015103816986084,
+      "logps/chosen": -9.364206314086914,
+      "logps/rejected": -10.420454025268555,
+      "loss": 0.5038,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.364206314086914,
+      "rewards/margins": 1.0562490224838257,
+      "rewards/rejected": -10.420454025268555,
+      "semantic_entropy": 0.0014547610189765692,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 17.51779749740211,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": 1.3948280811309814,
+      "logits/rejected": 1.4418094158172607,
+      "logps/chosen": -9.139997482299805,
+      "logps/rejected": -10.151327133178711,
+      "loss": 0.4559,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.139997482299805,
+      "rewards/margins": 1.0113298892974854,
+      "rewards/rejected": -10.151327133178711,
+      "semantic_entropy": 0.0017655401024967432,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 18.07511047903328,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": 1.3340023756027222,
+      "logits/rejected": 1.4303041696548462,
+      "logps/chosen": -9.04524040222168,
+      "logps/rejected": -10.115513801574707,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.04524040222168,
+      "rewards/margins": 1.0702718496322632,
+      "rewards/rejected": -10.115513801574707,
+      "semantic_entropy": 0.0020761829800903797,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 20.456562467372404,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": 1.406008005142212,
+      "logits/rejected": 1.4305920600891113,
+      "logps/chosen": -9.024477005004883,
+      "logps/rejected": -9.870010375976562,
+      "loss": 0.5182,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.024477005004883,
+      "rewards/margins": 0.8455320596694946,
+      "rewards/rejected": -9.870010375976562,
+      "semantic_entropy": 0.001961994683369994,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 22.572610590561663,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": 1.4253098964691162,
+      "logits/rejected": 1.4556143283843994,
+      "logps/chosen": -8.978473663330078,
+      "logps/rejected": -9.857033729553223,
+      "loss": 0.5809,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.978473663330078,
+      "rewards/margins": 0.8785587549209595,
+      "rewards/rejected": -9.857033729553223,
+      "semantic_entropy": 0.0024560727179050446,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 19.16776299258833,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": 1.422959566116333,
+      "logits/rejected": 1.489323616027832,
+      "logps/chosen": -9.024828910827637,
+      "logps/rejected": -9.809757232666016,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.024828910827637,
+      "rewards/margins": 0.7849278450012207,
+      "rewards/rejected": -9.809757232666016,
+      "semantic_entropy": 0.002740757539868355,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 20.452972527789843,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": 1.381536602973938,
+      "logits/rejected": 1.4693284034729004,
+      "logps/chosen": -8.905204772949219,
+      "logps/rejected": -9.88813304901123,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.905204772949219,
+      "rewards/margins": 0.982927680015564,
+      "rewards/rejected": -9.88813304901123,
+      "semantic_entropy": 0.0020987526513636112,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 24.051268813350163,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": 1.3987592458724976,
+      "logits/rejected": 1.5059475898742676,
+      "logps/chosen": -8.96232795715332,
+      "logps/rejected": -9.873229026794434,
+      "loss": 0.493,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.96232795715332,
+      "rewards/margins": 0.9109012484550476,
+      "rewards/rejected": -9.873229026794434,
+      "semantic_entropy": 0.0018539393786340952,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 17.95190535363776,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": 1.370980143547058,
+      "logits/rejected": 1.4280784130096436,
+      "logps/chosen": -8.974889755249023,
+      "logps/rejected": -9.99614429473877,
+      "loss": 0.4558,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.974889755249023,
+      "rewards/margins": 1.021254301071167,
+      "rewards/rejected": -9.99614429473877,
+      "semantic_entropy": 0.0019631218165159225,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 21.51685445563494,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": 1.3515079021453857,
+      "logits/rejected": 1.4074007272720337,
+      "logps/chosen": -8.987714767456055,
+      "logps/rejected": -10.103464126586914,
+      "loss": 0.4208,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -8.987714767456055,
+      "rewards/margins": 1.1157491207122803,
+      "rewards/rejected": -10.103464126586914,
+      "semantic_entropy": 0.0022278004325926304,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 14.850099487668794,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": 1.3546864986419678,
+      "logits/rejected": 1.4169265031814575,
+      "logps/chosen": -9.000819206237793,
+      "logps/rejected": -10.06425952911377,
+      "loss": 0.4665,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.000819206237793,
+      "rewards/margins": 1.063438892364502,
+      "rewards/rejected": -10.06425952911377,
+      "semantic_entropy": 0.002213912783190608,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 19.621345535251827,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": 1.341818928718567,
+      "logits/rejected": 1.3863704204559326,
+      "logps/chosen": -9.05766487121582,
+      "logps/rejected": -9.982002258300781,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.05766487121582,
+      "rewards/margins": 0.9243367910385132,
+      "rewards/rejected": -9.982002258300781,
+      "semantic_entropy": 0.0020083007402718067,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 18.618193613493,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": 1.3326647281646729,
+      "logits/rejected": 1.4035403728485107,
+      "logps/chosen": -9.093528747558594,
+      "logps/rejected": -10.121112823486328,
+      "loss": 0.4491,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.093528747558594,
+      "rewards/margins": 1.027583360671997,
+      "rewards/rejected": -10.121112823486328,
+      "semantic_entropy": 0.0019169582519680262,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 20.724085122328663,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": 1.3801498413085938,
+      "logits/rejected": 1.4021596908569336,
+      "logps/chosen": -9.225045204162598,
+      "logps/rejected": -10.090117454528809,
+      "loss": 0.5168,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.225045204162598,
+      "rewards/margins": 0.8650735020637512,
+      "rewards/rejected": -10.090117454528809,
+      "semantic_entropy": 0.0016959721688181162,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 24.823829183985556,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": 1.3034473657608032,
+      "logits/rejected": 1.3599941730499268,
+      "logps/chosen": -9.217083930969238,
+      "logps/rejected": -10.082822799682617,
+      "loss": 0.5043,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.217083930969238,
+      "rewards/margins": 0.8657392263412476,
+      "rewards/rejected": -10.082822799682617,
+      "semantic_entropy": 0.0019541464280337095,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 19.944474375705187,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": 1.3357312679290771,
+      "logits/rejected": 1.405626654624939,
+      "logps/chosen": -8.99936294555664,
+      "logps/rejected": -10.188141822814941,
+      "loss": 0.4477,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -8.99936294555664,
+      "rewards/margins": 1.1887791156768799,
+      "rewards/rejected": -10.188141822814941,
+      "semantic_entropy": 0.0019770406652241945,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 24.94967383919948,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": 1.312457799911499,
+      "logits/rejected": 1.3620131015777588,
+      "logps/chosen": -9.154726028442383,
+      "logps/rejected": -10.238590240478516,
+      "loss": 0.4629,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.154726028442383,
+      "rewards/margins": 1.083863377571106,
+      "rewards/rejected": -10.238590240478516,
+      "semantic_entropy": 0.0024866159074008465,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 17.329466375681626,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": 1.2930995225906372,
+      "logits/rejected": 1.3718712329864502,
+      "logps/chosen": -9.117254257202148,
+      "logps/rejected": -10.192098617553711,
+      "loss": 0.469,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.117254257202148,
+      "rewards/margins": 1.0748441219329834,
+      "rewards/rejected": -10.192098617553711,
+      "semantic_entropy": 0.001999816857278347,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 26.22909296597799,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": 1.3936941623687744,
+      "logits/rejected": 1.3912875652313232,
+      "logps/chosen": -9.185063362121582,
+      "logps/rejected": -10.229649543762207,
+      "loss": 0.4627,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.185063362121582,
+      "rewards/margins": 1.044585943222046,
+      "rewards/rejected": -10.229649543762207,
+      "semantic_entropy": 0.0018999949097633362,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 19.371015271415875,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": 1.322726845741272,
+      "logits/rejected": 1.403104543685913,
+      "logps/chosen": -9.25002384185791,
+      "logps/rejected": -10.093324661254883,
+      "loss": 0.528,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.25002384185791,
+      "rewards/margins": 0.8433005213737488,
+      "rewards/rejected": -10.093324661254883,
+      "semantic_entropy": 0.0017607096815481782,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 21.77626856921196,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": 1.3551298379898071,
+      "logits/rejected": 1.3962130546569824,
+      "logps/chosen": -9.298103332519531,
+      "logps/rejected": -10.353917121887207,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.298103332519531,
+      "rewards/margins": 1.0558137893676758,
+      "rewards/rejected": -10.353917121887207,
+      "semantic_entropy": 0.00168662890791893,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 24.544862105577508,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": 1.3699164390563965,
+      "logits/rejected": 1.4380617141723633,
+      "logps/chosen": -9.351114273071289,
+      "logps/rejected": -10.126466751098633,
+      "loss": 0.564,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -9.351114273071289,
+      "rewards/margins": 0.7753542065620422,
+      "rewards/rejected": -10.126466751098633,
+      "semantic_entropy": 0.0015395357040688396,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 21.96491561266928,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": 1.4216992855072021,
+      "logits/rejected": 1.4719842672348022,
+      "logps/chosen": -9.366571426391602,
+      "logps/rejected": -10.436944007873535,
+      "loss": 0.4742,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.366571426391602,
+      "rewards/margins": 1.0703723430633545,
+      "rewards/rejected": -10.436944007873535,
+      "semantic_entropy": 0.0015947825741022825,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 24.198905588013773,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": 1.3597904443740845,
+      "logits/rejected": 1.380866289138794,
+      "logps/chosen": -9.366788864135742,
+      "logps/rejected": -10.01229476928711,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.366788864135742,
+      "rewards/margins": 0.6455041766166687,
+      "rewards/rejected": -10.01229476928711,
+      "semantic_entropy": 0.0016107909614220262,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 15.212589101782092,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": 1.366815209388733,
+      "logits/rejected": 1.4347606897354126,
+      "logps/chosen": -9.34277057647705,
+      "logps/rejected": -10.516633987426758,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.34277057647705,
+      "rewards/margins": 1.1738636493682861,
+      "rewards/rejected": -10.516633987426758,
+      "semantic_entropy": 0.0015777539229020476,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 19.30021485531395,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": 1.3779293298721313,
+      "logits/rejected": 1.4349298477172852,
+      "logps/chosen": -9.422135353088379,
+      "logps/rejected": -10.354425430297852,
+      "loss": 0.5023,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.422135353088379,
+      "rewards/margins": 0.9322904348373413,
+      "rewards/rejected": -10.354425430297852,
+      "semantic_entropy": 0.001872875145636499,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 20.08868485221461,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": 1.4652825593948364,
+      "logits/rejected": 1.5223023891448975,
+      "logps/chosen": -9.617635726928711,
+      "logps/rejected": -10.569530487060547,
+      "loss": 0.5379,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.617635726928711,
+      "rewards/margins": 0.9518947601318359,
+      "rewards/rejected": -10.569530487060547,
+      "semantic_entropy": 0.0013659660471603274,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": 1.577815294265747,
+      "eval_logits/rejected": 1.6241698265075684,
+      "eval_logps/chosen": -9.708633422851562,
+      "eval_logps/rejected": -10.68848991394043,
+      "eval_loss": 0.5329428911209106,
+      "eval_rewards/accuracies": 0.715133547782898,
+      "eval_rewards/chosen": -9.708633422851562,
+      "eval_rewards/margins": 0.9798569679260254,
+      "eval_rewards/rejected": -10.68848991394043,
+      "eval_runtime": 34.8165,
+      "eval_samples_per_second": 38.631,
+      "eval_semantic_entropy": 0.0013822205364704132,
+      "eval_steps_per_second": 9.679,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 16.644693917546608,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": 1.4056084156036377,
+      "logits/rejected": 1.469939947128296,
+      "logps/chosen": -9.641395568847656,
+      "logps/rejected": -10.706151962280273,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.641395568847656,
+      "rewards/margins": 1.0647571086883545,
+      "rewards/rejected": -10.706151962280273,
+      "semantic_entropy": 0.0013048909604549408,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 22.939925014290832,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": 1.3790923357009888,
+      "logits/rejected": 1.4394371509552002,
+      "logps/chosen": -9.548755645751953,
+      "logps/rejected": -10.433923721313477,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.548755645751953,
+      "rewards/margins": 0.8851673007011414,
+      "rewards/rejected": -10.433923721313477,
+      "semantic_entropy": 0.0016102034132927656,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 21.577358724649326,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": 1.4795191287994385,
+      "logits/rejected": 1.5361725091934204,
+      "logps/chosen": -9.640555381774902,
+      "logps/rejected": -10.700790405273438,
+      "loss": 0.4525,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.640555381774902,
+      "rewards/margins": 1.0602333545684814,
+      "rewards/rejected": -10.700790405273438,
+      "semantic_entropy": 0.0013893931172788143,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 18.899427590423162,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": 1.4145498275756836,
+      "logits/rejected": 1.5122010707855225,
+      "logps/chosen": -9.888641357421875,
+      "logps/rejected": -10.90507698059082,
+      "loss": 0.5274,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.888641357421875,
+      "rewards/margins": 1.0164356231689453,
+      "rewards/rejected": -10.90507698059082,
+      "semantic_entropy": 0.0011320734629407525,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 19.66331409016772,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": 1.4619033336639404,
+      "logits/rejected": 1.5294215679168701,
+      "logps/chosen": -9.455495834350586,
+      "logps/rejected": -10.435941696166992,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.455495834350586,
+      "rewards/margins": 0.9804443120956421,
+      "rewards/rejected": -10.435941696166992,
+      "semantic_entropy": 0.0014900127425789833,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 26.18613064360708,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": 1.4180997610092163,
+      "logits/rejected": 1.4933340549468994,
+      "logps/chosen": -9.503206253051758,
+      "logps/rejected": -10.735602378845215,
+      "loss": 0.4475,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.503206253051758,
+      "rewards/margins": 1.2323966026306152,
+      "rewards/rejected": -10.735602378845215,
+      "semantic_entropy": 0.0016708982875570655,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 24.232463946626584,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": 1.432334065437317,
+      "logits/rejected": 1.4812839031219482,
+      "logps/chosen": -9.435246467590332,
+      "logps/rejected": -10.504186630249023,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.435246467590332,
+      "rewards/margins": 1.068939208984375,
+      "rewards/rejected": -10.504186630249023,
+      "semantic_entropy": 0.0017607705667614937,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 20.471224879096873,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": 1.4386165142059326,
+      "logits/rejected": 1.5068000555038452,
+      "logps/chosen": -9.584053993225098,
+      "logps/rejected": -10.751504898071289,
+      "loss": 0.4432,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.584053993225098,
+      "rewards/margins": 1.167452096939087,
+      "rewards/rejected": -10.751504898071289,
+      "semantic_entropy": 0.001337358495220542,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 23.21083128450944,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": 1.3803538084030151,
+      "logits/rejected": 1.432745337486267,
+      "logps/chosen": -9.65900707244873,
+      "logps/rejected": -10.629631042480469,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.65900707244873,
+      "rewards/margins": 0.9706231355667114,
+      "rewards/rejected": -10.629631042480469,
+      "semantic_entropy": 0.001433167839422822,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 21.702068635745672,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": 1.4878456592559814,
+      "logits/rejected": 1.5939511060714722,
+      "logps/chosen": -9.618230819702148,
+      "logps/rejected": -10.727155685424805,
+      "loss": 0.4577,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.618230819702148,
+      "rewards/margins": 1.1089264154434204,
+      "rewards/rejected": -10.727155685424805,
+      "semantic_entropy": 0.0013435513246804476,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 15.570251208492852,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": 1.4111377000808716,
+      "logits/rejected": 1.4851337671279907,
+      "logps/chosen": -9.336917877197266,
+      "logps/rejected": -10.429855346679688,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.336917877197266,
+      "rewards/margins": 1.092936635017395,
+      "rewards/rejected": -10.429855346679688,
+      "semantic_entropy": 0.001678326865658164,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 25.327077224246977,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": 1.3557472229003906,
+      "logits/rejected": 1.448624610900879,
+      "logps/chosen": -9.40028190612793,
+      "logps/rejected": -10.49830150604248,
+      "loss": 0.458,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.40028190612793,
+      "rewards/margins": 1.0980188846588135,
+      "rewards/rejected": -10.49830150604248,
+      "semantic_entropy": 0.0017656737472862005,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 29.333027459722697,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": 1.3939626216888428,
+      "logits/rejected": 1.4838975667953491,
+      "logps/chosen": -9.445150375366211,
+      "logps/rejected": -10.48935604095459,
+      "loss": 0.492,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.445150375366211,
+      "rewards/margins": 1.0442060232162476,
+      "rewards/rejected": -10.48935604095459,
+      "semantic_entropy": 0.0014688344672322273,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 24.198507618198587,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": 1.3835937976837158,
+      "logits/rejected": 1.4497041702270508,
+      "logps/chosen": -9.403009414672852,
+      "logps/rejected": -10.426477432250977,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.403009414672852,
+      "rewards/margins": 1.0234692096710205,
+      "rewards/rejected": -10.426477432250977,
+      "semantic_entropy": 0.001596631482243538,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 20.76827984015358,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": 1.3575257062911987,
+      "logits/rejected": 1.50045907497406,
+      "logps/chosen": -9.250945091247559,
+      "logps/rejected": -10.376054763793945,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.250945091247559,
+      "rewards/margins": 1.1251087188720703,
+      "rewards/rejected": -10.376054763793945,
+      "semantic_entropy": 0.0023337206803262234,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 29.196323121246397,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": 1.3501508235931396,
+      "logits/rejected": 1.412285327911377,
+      "logps/chosen": -9.150228500366211,
+      "logps/rejected": -10.251439094543457,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.150228500366211,
+      "rewards/margins": 1.1012094020843506,
+      "rewards/rejected": -10.251439094543457,
+      "semantic_entropy": 0.001867181621491909,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 25.702452826901656,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": 1.3400838375091553,
+      "logits/rejected": 1.4452638626098633,
+      "logps/chosen": -9.301922798156738,
+      "logps/rejected": -10.359292030334473,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.301922798156738,
+      "rewards/margins": 1.0573694705963135,
+      "rewards/rejected": -10.359292030334473,
+      "semantic_entropy": 0.0016484124353155494,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 18.439201760933702,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": 1.3446635007858276,
+      "logits/rejected": 1.421607255935669,
+      "logps/chosen": -9.198443412780762,
+      "logps/rejected": -10.281476974487305,
+      "loss": 0.4369,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.198443412780762,
+      "rewards/margins": 1.0830328464508057,
+      "rewards/rejected": -10.281476974487305,
+      "semantic_entropy": 0.0016971271252259612,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 22.651646413993397,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": 1.3639581203460693,
+      "logits/rejected": 1.4282277822494507,
+      "logps/chosen": -9.261232376098633,
+      "logps/rejected": -10.260461807250977,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.261232376098633,
+      "rewards/margins": 0.9992296099662781,
+      "rewards/rejected": -10.260461807250977,
+      "semantic_entropy": 0.0019416266586631536,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 26.126449208133266,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": 1.2474181652069092,
+      "logits/rejected": 1.3218269348144531,
+      "logps/chosen": -9.20488166809082,
+      "logps/rejected": -10.28109073638916,
+      "loss": 0.5168,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.20488166809082,
+      "rewards/margins": 1.0762097835540771,
+      "rewards/rejected": -10.28109073638916,
+      "semantic_entropy": 0.002146144863218069,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 27.884921780268872,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": 1.2788739204406738,
+      "logits/rejected": 1.3864109516143799,
+      "logps/chosen": -9.083578109741211,
+      "logps/rejected": -10.100305557250977,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.083578109741211,
+      "rewards/margins": 1.0167278051376343,
+      "rewards/rejected": -10.100305557250977,
+      "semantic_entropy": 0.0020475159399211407,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 20.63681149835847,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": 1.390315055847168,
+      "logits/rejected": 1.423758864402771,
+      "logps/chosen": -9.159515380859375,
+      "logps/rejected": -10.119551658630371,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.159515380859375,
+      "rewards/margins": 0.9600363969802856,
+      "rewards/rejected": -10.119551658630371,
+      "semantic_entropy": 0.0018103349721059203,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 27.290431172528233,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": 1.320866346359253,
+      "logits/rejected": 1.4118013381958008,
+      "logps/chosen": -9.175016403198242,
+      "logps/rejected": -10.129505157470703,
+      "loss": 0.5017,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.175016403198242,
+      "rewards/margins": 0.9544878005981445,
+      "rewards/rejected": -10.129505157470703,
+      "semantic_entropy": 0.0018231229623779655,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 16.005873839489738,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": 1.353188157081604,
+      "logits/rejected": 1.4561371803283691,
+      "logps/chosen": -9.009169578552246,
+      "logps/rejected": -10.238008499145508,
+      "loss": 0.4786,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.009169578552246,
+      "rewards/margins": 1.2288395166397095,
+      "rewards/rejected": -10.238008499145508,
+      "semantic_entropy": 0.0020765371154993773,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 21.254529711434365,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": 1.2880394458770752,
+      "logits/rejected": 1.3751757144927979,
+      "logps/chosen": -9.033915519714355,
+      "logps/rejected": -10.06599235534668,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.033915519714355,
+      "rewards/margins": 1.0320760011672974,
+      "rewards/rejected": -10.06599235534668,
+      "semantic_entropy": 0.002233104081824422,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 23.308757543307724,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": 1.3451558351516724,
+      "logits/rejected": 1.39243483543396,
+      "logps/chosen": -9.03105354309082,
+      "logps/rejected": -10.204160690307617,
+      "loss": 0.4502,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.03105354309082,
+      "rewards/margins": 1.1731078624725342,
+      "rewards/rejected": -10.204160690307617,
+      "semantic_entropy": 0.0022996277548372746,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 40.094518028135695,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": 1.3114280700683594,
+      "logits/rejected": 1.3928693532943726,
+      "logps/chosen": -9.359891891479492,
+      "logps/rejected": -10.244556427001953,
+      "loss": 0.5581,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.359891891479492,
+      "rewards/margins": 0.8846640586853027,
+      "rewards/rejected": -10.244556427001953,
+      "semantic_entropy": 0.002154856687411666,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 24.110208171469637,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": 1.3578107357025146,
+      "logits/rejected": 1.4353245496749878,
+      "logps/chosen": -9.060928344726562,
+      "logps/rejected": -10.32638168334961,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.060928344726562,
+      "rewards/margins": 1.2654531002044678,
+      "rewards/rejected": -10.32638168334961,
+      "semantic_entropy": 0.002143404446542263,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 19.856605421705808,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": 1.2724863290786743,
+      "logits/rejected": 1.3613078594207764,
+      "logps/chosen": -8.909154891967773,
+      "logps/rejected": -10.01855182647705,
+      "loss": 0.4531,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.909154891967773,
+      "rewards/margins": 1.10939621925354,
+      "rewards/rejected": -10.01855182647705,
+      "semantic_entropy": 0.002211230108514428,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 16.143329215646453,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": 1.2403380870819092,
+      "logits/rejected": 1.3354804515838623,
+      "logps/chosen": -9.096298217773438,
+      "logps/rejected": -10.119039535522461,
+      "loss": 0.4869,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.096298217773438,
+      "rewards/margins": 1.0227419137954712,
+      "rewards/rejected": -10.119039535522461,
+      "semantic_entropy": 0.00206415425054729,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 24.817853066155703,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": 1.3030577898025513,
+      "logits/rejected": 1.3561296463012695,
+      "logps/chosen": -9.10371208190918,
+      "logps/rejected": -10.018545150756836,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.10371208190918,
+      "rewards/margins": 0.9148343205451965,
+      "rewards/rejected": -10.018545150756836,
+      "semantic_entropy": 0.0019369969377294183,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 23.497675167682758,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": 1.206807255744934,
+      "logits/rejected": 1.2995214462280273,
+      "logps/chosen": -8.71178150177002,
+      "logps/rejected": -9.776965141296387,
+      "loss": 0.5058,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -8.71178150177002,
+      "rewards/margins": 1.0651828050613403,
+      "rewards/rejected": -9.776965141296387,
+      "semantic_entropy": 0.0027639290783554316,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 20.682618907452795,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": 1.2412288188934326,
+      "logits/rejected": 1.337101697921753,
+      "logps/chosen": -8.977112770080566,
+      "logps/rejected": -9.855443954467773,
+      "loss": 0.5238,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.977112770080566,
+      "rewards/margins": 0.878332257270813,
+      "rewards/rejected": -9.855443954467773,
+      "semantic_entropy": 0.0023172274231910706,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 23.35759608562459,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": 1.3106896877288818,
+      "logits/rejected": 1.384941816329956,
+      "logps/chosen": -9.11836051940918,
+      "logps/rejected": -9.855081558227539,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.11836051940918,
+      "rewards/margins": 0.7367221713066101,
+      "rewards/rejected": -9.855081558227539,
+      "semantic_entropy": 0.0019736960530281067,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 24.440253893361543,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": 1.3553003072738647,
+      "logits/rejected": 1.4123023748397827,
+      "logps/chosen": -9.361505508422852,
+      "logps/rejected": -10.256139755249023,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.361505508422852,
+      "rewards/margins": 0.8946337699890137,
+      "rewards/rejected": -10.256139755249023,
+      "semantic_entropy": 0.0015576521400362253,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 17.60362552102282,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": 1.3138495683670044,
+      "logits/rejected": 1.410321593284607,
+      "logps/chosen": -9.266050338745117,
+      "logps/rejected": -10.286043167114258,
+      "loss": 0.4912,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.266050338745117,
+      "rewards/margins": 1.0199923515319824,
+      "rewards/rejected": -10.286043167114258,
+      "semantic_entropy": 0.0015853249933570623,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 18.25107556907762,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": 1.2855336666107178,
+      "logits/rejected": 1.3057645559310913,
+      "logps/chosen": -9.303617477416992,
+      "logps/rejected": -10.235923767089844,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.303617477416992,
+      "rewards/margins": 0.9323053359985352,
+      "rewards/rejected": -10.235923767089844,
+      "semantic_entropy": 0.0015810371842235327,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 24.494927633961918,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": 1.364070177078247,
+      "logits/rejected": 1.4303703308105469,
+      "logps/chosen": -9.483635902404785,
+      "logps/rejected": -10.342909812927246,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.483635902404785,
+      "rewards/margins": 0.8592745065689087,
+      "rewards/rejected": -10.342909812927246,
+      "semantic_entropy": 0.0014347798423841596,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 21.67574478270861,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": 1.3466980457305908,
+      "logits/rejected": 1.413402795791626,
+      "logps/chosen": -9.331016540527344,
+      "logps/rejected": -10.395356178283691,
+      "loss": 0.4531,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.331016540527344,
+      "rewards/margins": 1.0643398761749268,
+      "rewards/rejected": -10.395356178283691,
+      "semantic_entropy": 0.0015169087564572692,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 22.133903285290035,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": 1.3628571033477783,
+      "logits/rejected": 1.4318151473999023,
+      "logps/chosen": -9.434852600097656,
+      "logps/rejected": -10.416900634765625,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.434852600097656,
+      "rewards/margins": 0.9820483922958374,
+      "rewards/rejected": -10.416900634765625,
+      "semantic_entropy": 0.0037233340553939342,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 23.22035034141245,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": 1.3545477390289307,
+      "logits/rejected": 1.4185240268707275,
+      "logps/chosen": -9.515121459960938,
+      "logps/rejected": -10.41678524017334,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.515121459960938,
+      "rewards/margins": 0.9016643762588501,
+      "rewards/rejected": -10.41678524017334,
+      "semantic_entropy": 0.001246383530087769,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 18.683421254627213,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": 1.3456099033355713,
+      "logits/rejected": 1.433081030845642,
+      "logps/chosen": -9.473508834838867,
+      "logps/rejected": -10.459129333496094,
+      "loss": 0.4677,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.473508834838867,
+      "rewards/margins": 0.9856211543083191,
+      "rewards/rejected": -10.459129333496094,
+      "semantic_entropy": 0.0018760154489427805,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 25.764188243340516,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": 1.3351361751556396,
+      "logits/rejected": 1.4037806987762451,
+      "logps/chosen": -9.325681686401367,
+      "logps/rejected": -10.336344718933105,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.325681686401367,
+      "rewards/margins": 1.0106641054153442,
+      "rewards/rejected": -10.336344718933105,
+      "semantic_entropy": 0.0016032636631280184,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 22.135172041219068,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": 1.4233031272888184,
+      "logits/rejected": 1.485378623008728,
+      "logps/chosen": -9.401700973510742,
+      "logps/rejected": -10.382223129272461,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.401700973510742,
+      "rewards/margins": 0.9805216789245605,
+      "rewards/rejected": -10.382223129272461,
+      "semantic_entropy": 0.0021838857792317867,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 19.263075141774117,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": 1.3283250331878662,
+      "logits/rejected": 1.3647691011428833,
+      "logps/chosen": -9.42963981628418,
+      "logps/rejected": -10.448928833007812,
+      "loss": 0.4879,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.42963981628418,
+      "rewards/margins": 1.019289493560791,
+      "rewards/rejected": -10.448928833007812,
+      "semantic_entropy": 0.001518217264674604,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 18.802828344868242,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": 1.3413537740707397,
+      "logits/rejected": 1.4202524423599243,
+      "logps/chosen": -9.357722282409668,
+      "logps/rejected": -10.62916374206543,
+      "loss": 0.4649,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.357722282409668,
+      "rewards/margins": 1.2714405059814453,
+      "rewards/rejected": -10.62916374206543,
+      "semantic_entropy": 0.001683186972513795,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 24.819496575586466,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": 1.3626841306686401,
+      "logits/rejected": 1.4237651824951172,
+      "logps/chosen": -9.39324951171875,
+      "logps/rejected": -10.46892261505127,
+      "loss": 0.4358,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.39324951171875,
+      "rewards/margins": 1.0756728649139404,
+      "rewards/rejected": -10.46892261505127,
+      "semantic_entropy": 0.0014191360678523779,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 21.80861958408374,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": 1.3689777851104736,
+      "logits/rejected": 1.412935495376587,
+      "logps/chosen": -9.392818450927734,
+      "logps/rejected": -10.384347915649414,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.392818450927734,
+      "rewards/margins": 0.9915293455123901,
+      "rewards/rejected": -10.384347915649414,
+      "semantic_entropy": 0.0015778569504618645,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 18.96906519357962,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": 1.3440065383911133,
+      "logits/rejected": 1.4519740343093872,
+      "logps/chosen": -9.319048881530762,
+      "logps/rejected": -10.432695388793945,
+      "loss": 0.4898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.319048881530762,
+      "rewards/margins": 1.113646388053894,
+      "rewards/rejected": -10.432695388793945,
+      "semantic_entropy": 0.0015323886182159185,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 19.87409135086091,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": 1.3444913625717163,
+      "logits/rejected": 1.4224002361297607,
+      "logps/chosen": -9.347864151000977,
+      "logps/rejected": -10.525368690490723,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.347864151000977,
+      "rewards/margins": 1.1775052547454834,
+      "rewards/rejected": -10.525368690490723,
+      "semantic_entropy": 0.0014065180439502,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 21.246142310902055,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": 1.355074167251587,
+      "logits/rejected": 1.453766942024231,
+      "logps/chosen": -9.477167129516602,
+      "logps/rejected": -10.51270580291748,
+      "loss": 0.497,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.477167129516602,
+      "rewards/margins": 1.0355393886566162,
+      "rewards/rejected": -10.51270580291748,
+      "semantic_entropy": 0.001408428535796702,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 19.16073457849523,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": 1.3953893184661865,
+      "logits/rejected": 1.4603233337402344,
+      "logps/chosen": -9.247777938842773,
+      "logps/rejected": -10.293495178222656,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.247777938842773,
+      "rewards/margins": 1.0457180738449097,
+      "rewards/rejected": -10.293495178222656,
+      "semantic_entropy": 0.0016628839075565338,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 23.84894151527458,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": 1.36844801902771,
+      "logits/rejected": 1.4255768060684204,
+      "logps/chosen": -9.340253829956055,
+      "logps/rejected": -10.320905685424805,
+      "loss": 0.505,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.340253829956055,
+      "rewards/margins": 0.9806511998176575,
+      "rewards/rejected": -10.320905685424805,
+      "semantic_entropy": 0.0014763828366994858,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 20.09543025883849,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": 1.334242582321167,
+      "logits/rejected": 1.4141027927398682,
+      "logps/chosen": -9.35572624206543,
+      "logps/rejected": -10.53090763092041,
+      "loss": 0.4539,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.35572624206543,
+      "rewards/margins": 1.175180196762085,
+      "rewards/rejected": -10.53090763092041,
+      "semantic_entropy": 0.0015369572211056948,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 21.195672643115945,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": 1.3593276739120483,
+      "logits/rejected": 1.3946046829223633,
+      "logps/chosen": -9.339418411254883,
+      "logps/rejected": -10.376855850219727,
+      "loss": 0.4761,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.339418411254883,
+      "rewards/margins": 1.037437915802002,
+      "rewards/rejected": -10.376855850219727,
+      "semantic_entropy": 0.0017293458804488182,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 28.34972013499806,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": 1.3700060844421387,
+      "logits/rejected": 1.4079654216766357,
+      "logps/chosen": -9.525930404663086,
+      "logps/rejected": -10.575078964233398,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.525930404663086,
+      "rewards/margins": 1.049149751663208,
+      "rewards/rejected": -10.575078964233398,
+      "semantic_entropy": 0.0013205587165430188,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 28.34319045086672,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": 1.3379520177841187,
+      "logits/rejected": 1.4237364530563354,
+      "logps/chosen": -9.519506454467773,
+      "logps/rejected": -10.528284072875977,
+      "loss": 0.5079,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.519506454467773,
+      "rewards/margins": 1.0087770223617554,
+      "rewards/rejected": -10.528284072875977,
+      "semantic_entropy": 0.001302249962463975,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 24.244902703252055,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": 1.396451711654663,
+      "logits/rejected": 1.4520469903945923,
+      "logps/chosen": -9.379877090454102,
+      "logps/rejected": -10.450872421264648,
+      "loss": 0.4528,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.379877090454102,
+      "rewards/margins": 1.0709969997406006,
+      "rewards/rejected": -10.450872421264648,
+      "semantic_entropy": 0.0016087412368506193,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 21.324270772877558,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": 1.3475533723831177,
+      "logits/rejected": 1.4435368776321411,
+      "logps/chosen": -9.439908027648926,
+      "logps/rejected": -10.302833557128906,
+      "loss": 0.4843,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.439908027648926,
+      "rewards/margins": 0.8629250526428223,
+      "rewards/rejected": -10.302833557128906,
+      "semantic_entropy": 0.001422605593688786,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 25.695686865725257,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": 1.3543260097503662,
+      "logits/rejected": 1.4266248941421509,
+      "logps/chosen": -9.244036674499512,
+      "logps/rejected": -10.314271926879883,
+      "loss": 0.4779,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.244036674499512,
+      "rewards/margins": 1.0702335834503174,
+      "rewards/rejected": -10.314271926879883,
+      "semantic_entropy": 0.0018264588434249163,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 28.38756153382181,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": 1.373712182044983,
+      "logits/rejected": 1.441104769706726,
+      "logps/chosen": -9.239967346191406,
+      "logps/rejected": -10.351564407348633,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.239967346191406,
+      "rewards/margins": 1.1115970611572266,
+      "rewards/rejected": -10.351564407348633,
+      "semantic_entropy": 0.0016596721252426505,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 23.01622743508027,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": 1.3101716041564941,
+      "logits/rejected": 1.3788301944732666,
+      "logps/chosen": -9.028715133666992,
+      "logps/rejected": -10.142562866210938,
+      "loss": 0.4511,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.028715133666992,
+      "rewards/margins": 1.1138484477996826,
+      "rewards/rejected": -10.142562866210938,
+      "semantic_entropy": 0.0021480643190443516,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 25.217640786656702,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": 1.3777731657028198,
+      "logits/rejected": 1.480454444885254,
+      "logps/chosen": -9.361075401306152,
+      "logps/rejected": -10.457845687866211,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.361075401306152,
+      "rewards/margins": 1.0967704057693481,
+      "rewards/rejected": -10.457845687866211,
+      "semantic_entropy": 0.001413219259120524,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 23.08176249156407,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": 1.307318925857544,
+      "logits/rejected": 1.377393126487732,
+      "logps/chosen": -9.370884895324707,
+      "logps/rejected": -10.413410186767578,
+      "loss": 0.5031,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.370884895324707,
+      "rewards/margins": 1.0425258874893188,
+      "rewards/rejected": -10.413410186767578,
+      "semantic_entropy": 0.00154262431897223,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 26.323772785842532,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": 1.371038556098938,
+      "logits/rejected": 1.4754021167755127,
+      "logps/chosen": -9.500500679016113,
+      "logps/rejected": -10.445425033569336,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.500500679016113,
+      "rewards/margins": 0.9449259638786316,
+      "rewards/rejected": -10.445425033569336,
+      "semantic_entropy": 0.0013014546129852533,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 35.38535167238163,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": 1.362022042274475,
+      "logits/rejected": 1.4099501371383667,
+      "logps/chosen": -9.43089771270752,
+      "logps/rejected": -10.350268363952637,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.43089771270752,
+      "rewards/margins": 0.9193711280822754,
+      "rewards/rejected": -10.350268363952637,
+      "semantic_entropy": 0.0015365493018180132,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 16.924307678879728,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": 1.3238650560379028,
+      "logits/rejected": 1.4098074436187744,
+      "logps/chosen": -9.22443962097168,
+      "logps/rejected": -10.410423278808594,
+      "loss": 0.4354,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.22443962097168,
+      "rewards/margins": 1.185984492301941,
+      "rewards/rejected": -10.410423278808594,
+      "semantic_entropy": 0.0017419023206457496,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 26.373548280716957,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": 1.2898046970367432,
+      "logits/rejected": 1.3914539813995361,
+      "logps/chosen": -9.366701126098633,
+      "logps/rejected": -10.193224906921387,
+      "loss": 0.531,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.366701126098633,
+      "rewards/margins": 0.8265244364738464,
+      "rewards/rejected": -10.193224906921387,
+      "semantic_entropy": 0.0015939153963699937,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 23.986956253838457,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": 1.286051869392395,
+      "logits/rejected": 1.423661470413208,
+      "logps/chosen": -9.197023391723633,
+      "logps/rejected": -10.26491928100586,
+      "loss": 0.4409,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.197023391723633,
+      "rewards/margins": 1.0678956508636475,
+      "rewards/rejected": -10.26491928100586,
+      "semantic_entropy": 0.0017132038483396173,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 22.323311281418736,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": 1.34428071975708,
+      "logits/rejected": 1.4011601209640503,
+      "logps/chosen": -9.382596969604492,
+      "logps/rejected": -10.220735549926758,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -9.382596969604492,
+      "rewards/margins": 0.838137149810791,
+      "rewards/rejected": -10.220735549926758,
+      "semantic_entropy": 0.0014858524082228541,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 16.48949455898873,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": 1.3016557693481445,
+      "logits/rejected": 1.3687784671783447,
+      "logps/chosen": -9.233752250671387,
+      "logps/rejected": -10.221487045288086,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.233752250671387,
+      "rewards/margins": 0.9877352714538574,
+      "rewards/rejected": -10.221487045288086,
+      "semantic_entropy": 0.0018190756672993302,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 25.79793801521153,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": 1.3280235528945923,
+      "logits/rejected": 1.368025302886963,
+      "logps/chosen": -9.21642017364502,
+      "logps/rejected": -10.163397789001465,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.21642017364502,
+      "rewards/margins": 0.9469785690307617,
+      "rewards/rejected": -10.163397789001465,
+      "semantic_entropy": 0.0017118094256147742,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 19.31913335096607,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": 1.3560396432876587,
+      "logits/rejected": 1.4198405742645264,
+      "logps/chosen": -9.240747451782227,
+      "logps/rejected": -10.421072006225586,
+      "loss": 0.4502,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.240747451782227,
+      "rewards/margins": 1.1803247928619385,
+      "rewards/rejected": -10.421072006225586,
+      "semantic_entropy": 0.0017440617084503174,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 20.635975052524465,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": 1.3265531063079834,
+      "logits/rejected": 1.4263205528259277,
+      "logps/chosen": -9.098268508911133,
+      "logps/rejected": -10.260825157165527,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.098268508911133,
+      "rewards/margins": 1.1625570058822632,
+      "rewards/rejected": -10.260825157165527,
+      "semantic_entropy": 0.0019632030744105577,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 18.36182661427557,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": 1.4008915424346924,
+      "logits/rejected": 1.4618009328842163,
+      "logps/chosen": -9.22640609741211,
+      "logps/rejected": -10.371707916259766,
+      "loss": 0.45,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.22640609741211,
+      "rewards/margins": 1.1452996730804443,
+      "rewards/rejected": -10.371707916259766,
+      "semantic_entropy": 0.001663418486714363,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 22.256764099259232,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": 1.4091641902923584,
+      "logits/rejected": 1.4529612064361572,
+      "logps/chosen": -9.310768127441406,
+      "logps/rejected": -10.365492820739746,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.310768127441406,
+      "rewards/margins": 1.0547243356704712,
+      "rewards/rejected": -10.365492820739746,
+      "semantic_entropy": 0.0016936215106397867,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 15.37201288863962,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": 1.420802116394043,
+      "logits/rejected": 1.46330726146698,
+      "logps/chosen": -9.584074020385742,
+      "logps/rejected": -10.759427070617676,
+      "loss": 0.4301,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.584074020385742,
+      "rewards/margins": 1.1753534078598022,
+      "rewards/rejected": -10.759427070617676,
+      "semantic_entropy": 0.0013953137677162886,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 20.650901391277134,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": 1.451188087463379,
+      "logits/rejected": 1.4849640130996704,
+      "logps/chosen": -9.653191566467285,
+      "logps/rejected": -10.58132553100586,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.653191566467285,
+      "rewards/margins": 0.928133487701416,
+      "rewards/rejected": -10.58132553100586,
+      "semantic_entropy": 0.0012009530328214169,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 15.469725661466901,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": 1.3754422664642334,
+      "logits/rejected": 1.436151146888733,
+      "logps/chosen": -9.677508354187012,
+      "logps/rejected": -10.746538162231445,
+      "loss": 0.4705,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.677508354187012,
+      "rewards/margins": 1.0690287351608276,
+      "rewards/rejected": -10.746538162231445,
+      "semantic_entropy": 0.001280719181522727,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 24.288523739185223,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": 1.3957017660140991,
+      "logits/rejected": 1.4477766752243042,
+      "logps/chosen": -9.76191234588623,
+      "logps/rejected": -10.680197715759277,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -9.76191234588623,
+      "rewards/margins": 0.9182855486869812,
+      "rewards/rejected": -10.680197715759277,
+      "semantic_entropy": 0.0012312938924878836,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": 1.4730833768844604,
+      "eval_logits/rejected": 1.5136833190917969,
+      "eval_logps/chosen": -9.640846252441406,
+      "eval_logps/rejected": -10.619263648986816,
+      "eval_loss": 0.5305958390235901,
+      "eval_rewards/accuracies": 0.721068263053894,
+      "eval_rewards/chosen": -9.640846252441406,
+      "eval_rewards/margins": 0.9784167408943176,
+      "eval_rewards/rejected": -10.619263648986816,
+      "eval_runtime": 34.777,
+      "eval_samples_per_second": 38.675,
+      "eval_semantic_entropy": 0.0014143266016617417,
+      "eval_steps_per_second": 9.69,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 25.47791665111295,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": 1.3983227014541626,
+      "logits/rejected": 1.419793725013733,
+      "logps/chosen": -9.479947090148926,
+      "logps/rejected": -10.457716941833496,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -9.479947090148926,
+      "rewards/margins": 0.977770209312439,
+      "rewards/rejected": -10.457716941833496,
+      "semantic_entropy": 0.0015946778003126383,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 18.461479891655696,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": 1.3343857526779175,
+      "logits/rejected": 1.408109188079834,
+      "logps/chosen": -9.384645462036133,
+      "logps/rejected": -10.405282974243164,
+      "loss": 0.4549,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.384645462036133,
+      "rewards/margins": 1.0206373929977417,
+      "rewards/rejected": -10.405282974243164,
+      "semantic_entropy": 0.0015872785588726401,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 21.237173288900287,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": 1.3513858318328857,
+      "logits/rejected": 1.4130383729934692,
+      "logps/chosen": -9.460577011108398,
+      "logps/rejected": -10.504884719848633,
+      "loss": 0.4858,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.460577011108398,
+      "rewards/margins": 1.0443065166473389,
+      "rewards/rejected": -10.504884719848633,
+      "semantic_entropy": 0.001385573879815638,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 20.78477666721928,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": 1.3560354709625244,
+      "logits/rejected": 1.3966361284255981,
+      "logps/chosen": -9.581274032592773,
+      "logps/rejected": -10.501707077026367,
+      "loss": 0.5136,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -9.581274032592773,
+      "rewards/margins": 0.9204346537590027,
+      "rewards/rejected": -10.501707077026367,
+      "semantic_entropy": 0.001438954728655517,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 19.105905590879505,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": 1.3557037115097046,
+      "logits/rejected": 1.3784071207046509,
+      "logps/chosen": -9.246512413024902,
+      "logps/rejected": -10.296817779541016,
+      "loss": 0.4572,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.246512413024902,
+      "rewards/margins": 1.0503051280975342,
+      "rewards/rejected": -10.296817779541016,
+      "semantic_entropy": 0.002138623734936118,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 20.79093164725505,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": 1.3022149801254272,
+      "logits/rejected": 1.3347588777542114,
+      "logps/chosen": -9.349907875061035,
+      "logps/rejected": -10.4881010055542,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.349907875061035,
+      "rewards/margins": 1.1381934881210327,
+      "rewards/rejected": -10.4881010055542,
+      "semantic_entropy": 0.0015870227944105864,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 25.989783292401242,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": 1.326825737953186,
+      "logits/rejected": 1.3874809741973877,
+      "logps/chosen": -9.351693153381348,
+      "logps/rejected": -10.509380340576172,
+      "loss": 0.4626,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.351693153381348,
+      "rewards/margins": 1.1576873064041138,
+      "rewards/rejected": -10.509380340576172,
+      "semantic_entropy": 0.0016175744822248816,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 21.187011376514448,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": 1.3296332359313965,
+      "logits/rejected": 1.3802952766418457,
+      "logps/chosen": -9.389993667602539,
+      "logps/rejected": -10.414243698120117,
+      "loss": 0.4595,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.389993667602539,
+      "rewards/margins": 1.0242516994476318,
+      "rewards/rejected": -10.414243698120117,
+      "semantic_entropy": 0.0017491963226348162,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 17.84373897140309,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": 1.298169732093811,
+      "logits/rejected": 1.3658256530761719,
+      "logps/chosen": -9.379101753234863,
+      "logps/rejected": -10.216226577758789,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.379101753234863,
+      "rewards/margins": 0.8371247053146362,
+      "rewards/rejected": -10.216226577758789,
+      "semantic_entropy": 0.0017547223251312971,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 22.57235907386148,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": 1.3423022031784058,
+      "logits/rejected": 1.3754068613052368,
+      "logps/chosen": -9.580296516418457,
+      "logps/rejected": -10.365510940551758,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.580296516418457,
+      "rewards/margins": 0.7852136492729187,
+      "rewards/rejected": -10.365510940551758,
+      "semantic_entropy": 0.0015635717427358031,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 31.28453615597256,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": 1.3620045185089111,
+      "logits/rejected": 1.4159082174301147,
+      "logps/chosen": -9.476283073425293,
+      "logps/rejected": -10.458658218383789,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.476283073425293,
+      "rewards/margins": 0.9823755025863647,
+      "rewards/rejected": -10.458658218383789,
+      "semantic_entropy": 0.0015389169566333294,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 26.812849879656582,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": 1.366742730140686,
+      "logits/rejected": 1.4117023944854736,
+      "logps/chosen": -9.505921363830566,
+      "logps/rejected": -10.415290832519531,
+      "loss": 0.5168,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.505921363830566,
+      "rewards/margins": 0.9093695878982544,
+      "rewards/rejected": -10.415290832519531,
+      "semantic_entropy": 0.0016773479292169213,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 16.780389796580895,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": 1.3662008047103882,
+      "logits/rejected": 1.4135236740112305,
+      "logps/chosen": -9.25890827178955,
+      "logps/rejected": -10.453153610229492,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.25890827178955,
+      "rewards/margins": 1.1942460536956787,
+      "rewards/rejected": -10.453153610229492,
+      "semantic_entropy": 0.0017207752680405974,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 24.58708400091573,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": 1.3937784433364868,
+      "logits/rejected": 1.4381487369537354,
+      "logps/chosen": -9.379903793334961,
+      "logps/rejected": -10.483904838562012,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.379903793334961,
+      "rewards/margins": 1.1040023565292358,
+      "rewards/rejected": -10.483904838562012,
+      "semantic_entropy": 0.0017203794559463859,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 16.937974096921728,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": 1.3459250926971436,
+      "logits/rejected": 1.4222503900527954,
+      "logps/chosen": -9.603610038757324,
+      "logps/rejected": -10.663354873657227,
+      "loss": 0.538,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.603610038757324,
+      "rewards/margins": 1.0597455501556396,
+      "rewards/rejected": -10.663354873657227,
+      "semantic_entropy": 0.0014383551897481084,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 17.16667030933467,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": 1.3834937810897827,
+      "logits/rejected": 1.4428292512893677,
+      "logps/chosen": -9.20801830291748,
+      "logps/rejected": -10.339136123657227,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.20801830291748,
+      "rewards/margins": 1.1311193704605103,
+      "rewards/rejected": -10.339136123657227,
+      "semantic_entropy": 0.0017466178396716714,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 27.032333530590186,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": 1.3401272296905518,
+      "logits/rejected": 1.3942828178405762,
+      "logps/chosen": -9.610759735107422,
+      "logps/rejected": -10.647703170776367,
+      "loss": 0.4778,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.610759735107422,
+      "rewards/margins": 1.0369436740875244,
+      "rewards/rejected": -10.647703170776367,
+      "semantic_entropy": 0.0014293697895482183,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 22.07342502319262,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": 1.4157532453536987,
+      "logits/rejected": 1.4793651103973389,
+      "logps/chosen": -9.63289737701416,
+      "logps/rejected": -10.604401588439941,
+      "loss": 0.509,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -9.63289737701416,
+      "rewards/margins": 0.9715040326118469,
+      "rewards/rejected": -10.604401588439941,
+      "semantic_entropy": 0.0014324572402983904,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 23.223101584567477,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": 1.378193974494934,
+      "logits/rejected": 1.4333282709121704,
+      "logps/chosen": -9.567280769348145,
+      "logps/rejected": -10.573023796081543,
+      "loss": 0.474,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -9.567280769348145,
+      "rewards/margins": 1.0057424306869507,
+      "rewards/rejected": -10.573023796081543,
+      "semantic_entropy": 0.0013292706571519375,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 25.373762435976154,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": 1.3801181316375732,
+      "logits/rejected": 1.447227954864502,
+      "logps/chosen": -9.463846206665039,
+      "logps/rejected": -10.421319961547852,
+      "loss": 0.5044,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -9.463846206665039,
+      "rewards/margins": 0.9574743509292603,
+      "rewards/rejected": -10.421319961547852,
+      "semantic_entropy": 0.0015416903188452125,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 24.90722508617924,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": 1.40720534324646,
+      "logits/rejected": 1.430706262588501,
+      "logps/chosen": -9.402141571044922,
+      "logps/rejected": -10.551558494567871,
+      "loss": 0.4573,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.402141571044922,
+      "rewards/margins": 1.1494184732437134,
+      "rewards/rejected": -10.551558494567871,
+      "semantic_entropy": 0.0016385046765208244,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 21.606035798721777,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": 1.4461241960525513,
+      "logits/rejected": 1.472377896308899,
+      "logps/chosen": -9.343709945678711,
+      "logps/rejected": -10.326766014099121,
+      "loss": 0.512,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.343709945678711,
+      "rewards/margins": 0.9830554127693176,
+      "rewards/rejected": -10.326766014099121,
+      "semantic_entropy": 0.0016572829335927963,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 19.819352651468066,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": 1.4013676643371582,
+      "logits/rejected": 1.4346959590911865,
+      "logps/chosen": -9.462570190429688,
+      "logps/rejected": -10.65794563293457,
+      "loss": 0.4491,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -9.462570190429688,
+      "rewards/margins": 1.1953752040863037,
+      "rewards/rejected": -10.65794563293457,
+      "semantic_entropy": 0.001646661781705916,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 21.423727444948984,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": 1.3792479038238525,
+      "logits/rejected": 1.4380064010620117,
+      "logps/chosen": -9.366070747375488,
+      "logps/rejected": -10.382570266723633,
+      "loss": 0.4697,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.366070747375488,
+      "rewards/margins": 1.016500473022461,
+      "rewards/rejected": -10.382570266723633,
+      "semantic_entropy": 0.001614117412827909,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 19.299899155699503,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": 1.3063713312149048,
+      "logits/rejected": 1.353844404220581,
+      "logps/chosen": -9.323602676391602,
+      "logps/rejected": -10.47291088104248,
+      "loss": 0.4483,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -9.323602676391602,
+      "rewards/margins": 1.149308204650879,
+      "rewards/rejected": -10.47291088104248,
+      "semantic_entropy": 0.0016060378402471542,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 21.765000814472653,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": 1.4049794673919678,
+      "logits/rejected": 1.4536964893341064,
+      "logps/chosen": -9.394552230834961,
+      "logps/rejected": -10.389281272888184,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.394552230834961,
+      "rewards/margins": 0.9947283864021301,
+      "rewards/rejected": -10.389281272888184,
+      "semantic_entropy": 0.0015563599299639463,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 25.686548645574337,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": 1.3877205848693848,
+      "logits/rejected": 1.44142484664917,
+      "logps/chosen": -9.436675071716309,
+      "logps/rejected": -10.481142044067383,
+      "loss": 0.5321,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -9.436675071716309,
+      "rewards/margins": 1.044466495513916,
+      "rewards/rejected": -10.481142044067383,
+      "semantic_entropy": 0.0015583592467010021,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 24.912262703697184,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": 1.3917453289031982,
+      "logits/rejected": 1.4299163818359375,
+      "logps/chosen": -9.417211532592773,
+      "logps/rejected": -10.372767448425293,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.417211532592773,
+      "rewards/margins": 0.9555555582046509,
+      "rewards/rejected": -10.372767448425293,
+      "semantic_entropy": 0.0017329473048448563,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 19.722058992675368,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": 1.361957311630249,
+      "logits/rejected": 1.4019243717193604,
+      "logps/chosen": -9.383630752563477,
+      "logps/rejected": -10.681212425231934,
+      "loss": 0.3883,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.383630752563477,
+      "rewards/margins": 1.2975822687149048,
+      "rewards/rejected": -10.681212425231934,
+      "semantic_entropy": 0.0024538571015000343,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 16.278332885714466,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": 1.3411544561386108,
+      "logits/rejected": 1.3961362838745117,
+      "logps/chosen": -9.219443321228027,
+      "logps/rejected": -10.508321762084961,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.219443321228027,
+      "rewards/margins": 1.2888777256011963,
+      "rewards/rejected": -10.508321762084961,
+      "semantic_entropy": 0.0021164563950151205,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 16.86541052313204,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": 1.363463282585144,
+      "logits/rejected": 1.4307656288146973,
+      "logps/chosen": -9.471110343933105,
+      "logps/rejected": -10.774835586547852,
+      "loss": 0.3931,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.471110343933105,
+      "rewards/margins": 1.3037267923355103,
+      "rewards/rejected": -10.774835586547852,
+      "semantic_entropy": 0.00154945719987154,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 15.074656001561323,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": 1.3614099025726318,
+      "logits/rejected": 1.406822919845581,
+      "logps/chosen": -9.651963233947754,
+      "logps/rejected": -11.084357261657715,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.651963233947754,
+      "rewards/margins": 1.4323933124542236,
+      "rewards/rejected": -11.084357261657715,
+      "semantic_entropy": 0.0016995972255244851,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 25.334243439603927,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": 1.3539074659347534,
+      "logits/rejected": 1.4531766176223755,
+      "logps/chosen": -9.801499366760254,
+      "logps/rejected": -11.03126335144043,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.801499366760254,
+      "rewards/margins": 1.2297637462615967,
+      "rewards/rejected": -11.03126335144043,
+      "semantic_entropy": 0.001252225716598332,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 25.869919190743595,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": 1.3088419437408447,
+      "logits/rejected": 1.3882651329040527,
+      "logps/chosen": -9.548527717590332,
+      "logps/rejected": -10.975045204162598,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.548527717590332,
+      "rewards/margins": 1.4265177249908447,
+      "rewards/rejected": -10.975045204162598,
+      "semantic_entropy": 0.0017250744858756661,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 24.935017439885012,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": 1.3819299936294556,
+      "logits/rejected": 1.4063533544540405,
+      "logps/chosen": -9.854742050170898,
+      "logps/rejected": -11.120920181274414,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.854742050170898,
+      "rewards/margins": 1.2661776542663574,
+      "rewards/rejected": -11.120920181274414,
+      "semantic_entropy": 0.0012488319771364331,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 19.568062040206325,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": 1.378584623336792,
+      "logits/rejected": 1.431771159172058,
+      "logps/chosen": -9.546972274780273,
+      "logps/rejected": -10.842710494995117,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.546972274780273,
+      "rewards/margins": 1.2957381010055542,
+      "rewards/rejected": -10.842710494995117,
+      "semantic_entropy": 0.0014592509251087904,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 18.38374226984599,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": 1.3396540880203247,
+      "logits/rejected": 1.4232468605041504,
+      "logps/chosen": -9.568887710571289,
+      "logps/rejected": -10.817168235778809,
+      "loss": 0.4464,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.568887710571289,
+      "rewards/margins": 1.2482802867889404,
+      "rewards/rejected": -10.817168235778809,
+      "semantic_entropy": 0.0015056414995342493,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 15.390657086706767,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": 1.2979118824005127,
+      "logits/rejected": 1.3755037784576416,
+      "logps/chosen": -9.517377853393555,
+      "logps/rejected": -10.936643600463867,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.517377853393555,
+      "rewards/margins": 1.4192655086517334,
+      "rewards/rejected": -10.936643600463867,
+      "semantic_entropy": 0.0015412901993840933,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 23.26226360760525,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": 1.3183434009552002,
+      "logits/rejected": 1.356791615486145,
+      "logps/chosen": -9.343511581420898,
+      "logps/rejected": -10.598734855651855,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.343511581420898,
+      "rewards/margins": 1.2552231550216675,
+      "rewards/rejected": -10.598734855651855,
+      "semantic_entropy": 0.0018636903259903193,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 26.309156130970585,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": 1.3136707544326782,
+      "logits/rejected": 1.3974666595458984,
+      "logps/chosen": -9.302316665649414,
+      "logps/rejected": -10.618941307067871,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.302316665649414,
+      "rewards/margins": 1.3166248798370361,
+      "rewards/rejected": -10.618941307067871,
+      "semantic_entropy": 0.002033089753240347,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 28.893294946359696,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": 1.375326156616211,
+      "logits/rejected": 1.4643354415893555,
+      "logps/chosen": -9.569734573364258,
+      "logps/rejected": -10.787467956542969,
+      "loss": 0.431,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.569734573364258,
+      "rewards/margins": 1.2177342176437378,
+      "rewards/rejected": -10.787467956542969,
+      "semantic_entropy": 0.0012809450272470713,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 20.40934615666426,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": 1.3504489660263062,
+      "logits/rejected": 1.4234578609466553,
+      "logps/chosen": -9.475091934204102,
+      "logps/rejected": -10.807382583618164,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.475091934204102,
+      "rewards/margins": 1.332289457321167,
+      "rewards/rejected": -10.807382583618164,
+      "semantic_entropy": 0.0014203010359779,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 18.398920795647303,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": 1.3088784217834473,
+      "logits/rejected": 1.3825223445892334,
+      "logps/chosen": -9.541278839111328,
+      "logps/rejected": -10.867883682250977,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.541278839111328,
+      "rewards/margins": 1.326603651046753,
+      "rewards/rejected": -10.867883682250977,
+      "semantic_entropy": 0.0018189729889854789,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 21.03870269877971,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": 1.3326836824417114,
+      "logits/rejected": 1.4420342445373535,
+      "logps/chosen": -9.807455062866211,
+      "logps/rejected": -11.009900093078613,
+      "loss": 0.4338,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.807455062866211,
+      "rewards/margins": 1.2024445533752441,
+      "rewards/rejected": -11.009900093078613,
+      "semantic_entropy": 0.0012127887457609177,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 20.049783042197546,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": 1.3785035610198975,
+      "logits/rejected": 1.425476312637329,
+      "logps/chosen": -9.440518379211426,
+      "logps/rejected": -10.80799674987793,
+      "loss": 0.3698,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -9.440518379211426,
+      "rewards/margins": 1.367479681968689,
+      "rewards/rejected": -10.80799674987793,
+      "semantic_entropy": 0.001606472535058856,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 24.38957069947189,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": 1.3370369672775269,
+      "logits/rejected": 1.4031522274017334,
+      "logps/chosen": -9.689762115478516,
+      "logps/rejected": -11.109864234924316,
+      "loss": 0.377,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.689762115478516,
+      "rewards/margins": 1.4201027154922485,
+      "rewards/rejected": -11.109864234924316,
+      "semantic_entropy": 0.0012739686062559485,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 15.608652774836429,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": 1.3895460367202759,
+      "logits/rejected": 1.475119948387146,
+      "logps/chosen": -9.639775276184082,
+      "logps/rejected": -11.111741065979004,
+      "loss": 0.3581,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.639775276184082,
+      "rewards/margins": 1.4719641208648682,
+      "rewards/rejected": -11.111741065979004,
+      "semantic_entropy": 0.0014504891587421298,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 21.59741140731758,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": 1.410625696182251,
+      "logits/rejected": 1.4873372316360474,
+      "logps/chosen": -9.431108474731445,
+      "logps/rejected": -10.768244743347168,
+      "loss": 0.4035,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.431108474731445,
+      "rewards/margins": 1.3371379375457764,
+      "rewards/rejected": -10.768244743347168,
+      "semantic_entropy": 0.0017133388901129365,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 16.073608659932667,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": 1.4232189655303955,
+      "logits/rejected": 1.5133533477783203,
+      "logps/chosen": -9.667218208312988,
+      "logps/rejected": -11.1137113571167,
+      "loss": 0.3511,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -9.667218208312988,
+      "rewards/margins": 1.446493148803711,
+      "rewards/rejected": -11.1137113571167,
+      "semantic_entropy": 0.001395820057950914,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 20.56206237957164,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": 1.3987468481063843,
+      "logits/rejected": 1.4912564754486084,
+      "logps/chosen": -9.583246231079102,
+      "logps/rejected": -10.706694602966309,
+      "loss": 0.4551,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.583246231079102,
+      "rewards/margins": 1.123448133468628,
+      "rewards/rejected": -10.706694602966309,
+      "semantic_entropy": 0.0013306590262800455,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 22.845317628064127,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": 1.4165273904800415,
+      "logits/rejected": 1.4965639114379883,
+      "logps/chosen": -9.417463302612305,
+      "logps/rejected": -10.942605018615723,
+      "loss": 0.4034,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.417463302612305,
+      "rewards/margins": 1.5251425504684448,
+      "rewards/rejected": -10.942605018615723,
+      "semantic_entropy": 0.0018000425770878792,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 24.34989579064759,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": 1.4361201524734497,
+      "logits/rejected": 1.4886738061904907,
+      "logps/chosen": -9.454949378967285,
+      "logps/rejected": -10.886585235595703,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.454949378967285,
+      "rewards/margins": 1.4316370487213135,
+      "rewards/rejected": -10.886585235595703,
+      "semantic_entropy": 0.001460518455132842,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 23.495564113280565,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": 1.3781983852386475,
+      "logits/rejected": 1.4820003509521484,
+      "logps/chosen": -9.403005599975586,
+      "logps/rejected": -10.801973342895508,
+      "loss": 0.3516,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -9.403005599975586,
+      "rewards/margins": 1.3989660739898682,
+      "rewards/rejected": -10.801973342895508,
+      "semantic_entropy": 0.001604907913133502,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 23.56794661021698,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": 1.380800485610962,
+      "logits/rejected": 1.4365824460983276,
+      "logps/chosen": -9.221662521362305,
+      "logps/rejected": -10.535917282104492,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.221662521362305,
+      "rewards/margins": 1.3142549991607666,
+      "rewards/rejected": -10.535917282104492,
+      "semantic_entropy": 0.0020305518992245197,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 22.763692878608765,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": 1.438855528831482,
+      "logits/rejected": 1.5193105936050415,
+      "logps/chosen": -9.517614364624023,
+      "logps/rejected": -10.867307662963867,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.517614364624023,
+      "rewards/margins": 1.3496941328048706,
+      "rewards/rejected": -10.867307662963867,
+      "semantic_entropy": 0.0016832739347591996,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 24.321140009446736,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": 1.4655249118804932,
+      "logits/rejected": 1.5615251064300537,
+      "logps/chosen": -9.267009735107422,
+      "logps/rejected": -10.817902565002441,
+      "loss": 0.3422,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.267009735107422,
+      "rewards/margins": 1.5508928298950195,
+      "rewards/rejected": -10.817902565002441,
+      "semantic_entropy": 0.001808484666980803,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 22.297167372509843,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": 1.4283634424209595,
+      "logits/rejected": 1.482744574546814,
+      "logps/chosen": -9.2850980758667,
+      "logps/rejected": -10.93457317352295,
+      "loss": 0.3511,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.2850980758667,
+      "rewards/margins": 1.6494756937026978,
+      "rewards/rejected": -10.93457317352295,
+      "semantic_entropy": 0.002099847886711359,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 26.692041871329543,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": 1.4510812759399414,
+      "logits/rejected": 1.4991604089736938,
+      "logps/chosen": -9.546110153198242,
+      "logps/rejected": -10.944421768188477,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.546110153198242,
+      "rewards/margins": 1.3983123302459717,
+      "rewards/rejected": -10.944421768188477,
+      "semantic_entropy": 0.001652482314966619,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 19.127272625695273,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": 1.4259823560714722,
+      "logits/rejected": 1.5467689037322998,
+      "logps/chosen": -9.627596855163574,
+      "logps/rejected": -11.090128898620605,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.627596855163574,
+      "rewards/margins": 1.462532639503479,
+      "rewards/rejected": -11.090128898620605,
+      "semantic_entropy": 0.0015671420842409134,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 21.75967892964534,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": 1.490490198135376,
+      "logits/rejected": 1.5376614332199097,
+      "logps/chosen": -9.559449195861816,
+      "logps/rejected": -10.979775428771973,
+      "loss": 0.3588,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -9.559449195861816,
+      "rewards/margins": 1.4203264713287354,
+      "rewards/rejected": -10.979775428771973,
+      "semantic_entropy": 0.0016358118737116456,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 22.58427231948585,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": 1.4735342264175415,
+      "logits/rejected": 1.5532333850860596,
+      "logps/chosen": -9.747026443481445,
+      "logps/rejected": -11.119292259216309,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.747026443481445,
+      "rewards/margins": 1.3722654581069946,
+      "rewards/rejected": -11.119292259216309,
+      "semantic_entropy": 0.001798037439584732,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 23.35991615736964,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": 1.4969531297683716,
+      "logits/rejected": 1.5728718042373657,
+      "logps/chosen": -9.619976997375488,
+      "logps/rejected": -10.950164794921875,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.619976997375488,
+      "rewards/margins": 1.3301880359649658,
+      "rewards/rejected": -10.950164794921875,
+      "semantic_entropy": 0.0013786833733320236,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 26.90653298273018,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": 1.453338861465454,
+      "logits/rejected": 1.5328700542449951,
+      "logps/chosen": -9.9006986618042,
+      "logps/rejected": -11.130319595336914,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -9.9006986618042,
+      "rewards/margins": 1.2296206951141357,
+      "rewards/rejected": -11.130319595336914,
+      "semantic_entropy": 0.0012676132610067725,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 22.728029600591825,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": 1.4504599571228027,
+      "logits/rejected": 1.5432534217834473,
+      "logps/chosen": -9.618690490722656,
+      "logps/rejected": -11.176097869873047,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.618690490722656,
+      "rewards/margins": 1.5574071407318115,
+      "rewards/rejected": -11.176097869873047,
+      "semantic_entropy": 0.0014000110095366836,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 19.527105644841843,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": 1.4657458066940308,
+      "logits/rejected": 1.521693229675293,
+      "logps/chosen": -9.680712699890137,
+      "logps/rejected": -11.186546325683594,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.680712699890137,
+      "rewards/margins": 1.505833625793457,
+      "rewards/rejected": -11.186546325683594,
+      "semantic_entropy": 0.0013229264877736568,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 20.043325222336087,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": 1.4538131952285767,
+      "logits/rejected": 1.5334721803665161,
+      "logps/chosen": -9.706262588500977,
+      "logps/rejected": -11.312582015991211,
+      "loss": 0.3765,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.706262588500977,
+      "rewards/margins": 1.6063206195831299,
+      "rewards/rejected": -11.312582015991211,
+      "semantic_entropy": 0.0013946216786280274,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 17.682550042101315,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": 1.4708583354949951,
+      "logits/rejected": 1.552058458328247,
+      "logps/chosen": -9.794490814208984,
+      "logps/rejected": -11.24831485748291,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.794490814208984,
+      "rewards/margins": 1.4538249969482422,
+      "rewards/rejected": -11.24831485748291,
+      "semantic_entropy": 0.001196379540488124,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 25.77273561842405,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": 1.4553499221801758,
+      "logits/rejected": 1.5022424459457397,
+      "logps/chosen": -9.798062324523926,
+      "logps/rejected": -11.340188980102539,
+      "loss": 0.3812,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.798062324523926,
+      "rewards/margins": 1.542126178741455,
+      "rewards/rejected": -11.340188980102539,
+      "semantic_entropy": 0.0013756358530372381,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 25.268711514488366,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": 1.437491774559021,
+      "logits/rejected": 1.496272325515747,
+      "logps/chosen": -9.945511817932129,
+      "logps/rejected": -10.97688102722168,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.945511817932129,
+      "rewards/margins": 1.0313698053359985,
+      "rewards/rejected": -10.97688102722168,
+      "semantic_entropy": 0.001087284879758954,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 27.60735266343803,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": 1.4982235431671143,
+      "logits/rejected": 1.5658185482025146,
+      "logps/chosen": -9.805229187011719,
+      "logps/rejected": -11.388738632202148,
+      "loss": 0.3538,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.805229187011719,
+      "rewards/margins": 1.5835106372833252,
+      "rewards/rejected": -11.388738632202148,
+      "semantic_entropy": 0.0015130144311115146,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 28.768125481424892,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": 1.435878872871399,
+      "logits/rejected": 1.5036174058914185,
+      "logps/chosen": -9.82420825958252,
+      "logps/rejected": -11.39389705657959,
+      "loss": 0.386,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.82420825958252,
+      "rewards/margins": 1.5696887969970703,
+      "rewards/rejected": -11.39389705657959,
+      "semantic_entropy": 0.0013821411412209272,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 15.736276741474644,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": 1.4873812198638916,
+      "logits/rejected": 1.5616713762283325,
+      "logps/chosen": -9.76807689666748,
+      "logps/rejected": -11.532633781433105,
+      "loss": 0.3308,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.76807689666748,
+      "rewards/margins": 1.7645561695098877,
+      "rewards/rejected": -11.532633781433105,
+      "semantic_entropy": 0.0014301531482487917,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 26.87785334707155,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": 1.5220239162445068,
+      "logits/rejected": 1.5978310108184814,
+      "logps/chosen": -10.030058860778809,
+      "logps/rejected": -11.635004997253418,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.030058860778809,
+      "rewards/margins": 1.604946494102478,
+      "rewards/rejected": -11.635004997253418,
+      "semantic_entropy": 0.0012983604101464152,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 22.945993894466923,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": 1.5306103229522705,
+      "logits/rejected": 1.596461534500122,
+      "logps/chosen": -9.908811569213867,
+      "logps/rejected": -11.430213928222656,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.908811569213867,
+      "rewards/margins": 1.5214024782180786,
+      "rewards/rejected": -11.430213928222656,
+      "semantic_entropy": 0.0013624271377921104,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 25.332674916630445,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": 1.5457031726837158,
+      "logits/rejected": 1.6189396381378174,
+      "logps/chosen": -9.670294761657715,
+      "logps/rejected": -11.195516586303711,
+      "loss": 0.3428,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -9.670294761657715,
+      "rewards/margins": 1.5252220630645752,
+      "rewards/rejected": -11.195516586303711,
+      "semantic_entropy": 0.0016205990687012672,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 25.290684109867332,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": 1.4930251836776733,
+      "logits/rejected": 1.5451852083206177,
+      "logps/chosen": -9.777398109436035,
+      "logps/rejected": -11.42778491973877,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.777398109436035,
+      "rewards/margins": 1.6503868103027344,
+      "rewards/rejected": -11.42778491973877,
+      "semantic_entropy": 0.0018663747468963265,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 25.7593680480595,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": 1.4861161708831787,
+      "logits/rejected": 1.5526365041732788,
+      "logps/chosen": -9.842808723449707,
+      "logps/rejected": -11.32755184173584,
+      "loss": 0.3512,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.842808723449707,
+      "rewards/margins": 1.4847428798675537,
+      "rewards/rejected": -11.32755184173584,
+      "semantic_entropy": 0.0012635341845452785,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 25.524293941281446,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": 1.4640657901763916,
+      "logits/rejected": 1.524431586265564,
+      "logps/chosen": -9.777196884155273,
+      "logps/rejected": -11.392870903015137,
+      "loss": 0.3369,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.777196884155273,
+      "rewards/margins": 1.6156740188598633,
+      "rewards/rejected": -11.392870903015137,
+      "semantic_entropy": 0.001448600785806775,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 26.118865661405305,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": 1.4528363943099976,
+      "logits/rejected": 1.561838150024414,
+      "logps/chosen": -10.050291061401367,
+      "logps/rejected": -11.513885498046875,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -10.050291061401367,
+      "rewards/margins": 1.4635932445526123,
+      "rewards/rejected": -11.513885498046875,
+      "semantic_entropy": 0.0011458104709163308,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 26.353416464883725,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": 1.5036729574203491,
+      "logits/rejected": 1.559648871421814,
+      "logps/chosen": -9.863225936889648,
+      "logps/rejected": -11.436901092529297,
+      "loss": 0.3725,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.863225936889648,
+      "rewards/margins": 1.573674201965332,
+      "rewards/rejected": -11.436901092529297,
+      "semantic_entropy": 0.0012434284435585141,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": 1.6275861263275146,
+      "eval_logits/rejected": 1.6740076541900635,
+      "eval_logps/chosen": -10.245865821838379,
+      "eval_logps/rejected": -11.388835906982422,
+      "eval_loss": 0.5394824147224426,
+      "eval_rewards/accuracies": 0.7240356206893921,
+      "eval_rewards/chosen": -10.245865821838379,
+      "eval_rewards/margins": 1.1429716348648071,
+      "eval_rewards/rejected": -11.388835906982422,
+      "eval_runtime": 34.8312,
+      "eval_samples_per_second": 38.615,
+      "eval_semantic_entropy": 0.001081341877579689,
+      "eval_steps_per_second": 9.675,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 26.68755459109246,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": 1.5227869749069214,
+      "logits/rejected": 1.613612174987793,
+      "logps/chosen": -10.205938339233398,
+      "logps/rejected": -11.593180656433105,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.205938339233398,
+      "rewards/margins": 1.387242078781128,
+      "rewards/rejected": -11.593180656433105,
+      "semantic_entropy": 0.0012021606089547276,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 26.013328784901525,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": 1.4837193489074707,
+      "logits/rejected": 1.5490175485610962,
+      "logps/chosen": -10.06013298034668,
+      "logps/rejected": -11.392491340637207,
+      "loss": 0.4406,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.06013298034668,
+      "rewards/margins": 1.3323566913604736,
+      "rewards/rejected": -11.392491340637207,
+      "semantic_entropy": 0.0010128960711881518,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 29.710327111346036,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": 1.5103034973144531,
+      "logits/rejected": 1.610754370689392,
+      "logps/chosen": -10.20878791809082,
+      "logps/rejected": -11.58086109161377,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.20878791809082,
+      "rewards/margins": 1.3720736503601074,
+      "rewards/rejected": -11.58086109161377,
+      "semantic_entropy": 0.0009880965808406472,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 17.11726424950132,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": 1.4395703077316284,
+      "logits/rejected": 1.529000997543335,
+      "logps/chosen": -10.053105354309082,
+      "logps/rejected": -11.596004486083984,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.053105354309082,
+      "rewards/margins": 1.5428993701934814,
+      "rewards/rejected": -11.596004486083984,
+      "semantic_entropy": 0.001334618078544736,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 35.10997062701881,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": 1.5506001710891724,
+      "logits/rejected": 1.5966923236846924,
+      "logps/chosen": -9.996886253356934,
+      "logps/rejected": -11.112903594970703,
+      "loss": 0.4697,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -9.996886253356934,
+      "rewards/margins": 1.116018295288086,
+      "rewards/rejected": -11.112903594970703,
+      "semantic_entropy": 0.0013364561600610614,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 17.829560061155753,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": 1.4738044738769531,
+      "logits/rejected": 1.54628586769104,
+      "logps/chosen": -9.820981979370117,
+      "logps/rejected": -11.45405387878418,
+      "loss": 0.3248,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.820981979370117,
+      "rewards/margins": 1.6330699920654297,
+      "rewards/rejected": -11.45405387878418,
+      "semantic_entropy": 0.001320987707003951,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 22.04458809145264,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": 1.4746648073196411,
+      "logits/rejected": 1.4956865310668945,
+      "logps/chosen": -9.836318016052246,
+      "logps/rejected": -11.114892959594727,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.836318016052246,
+      "rewards/margins": 1.27857506275177,
+      "rewards/rejected": -11.114892959594727,
+      "semantic_entropy": 0.0011716438457369804,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 29.46897780300108,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": 1.5174767971038818,
+      "logits/rejected": 1.5269310474395752,
+      "logps/chosen": -9.938007354736328,
+      "logps/rejected": -11.331061363220215,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.938007354736328,
+      "rewards/margins": 1.393054723739624,
+      "rewards/rejected": -11.331061363220215,
+      "semantic_entropy": 0.0013201211113482714,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 23.542792559123583,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": 1.5060899257659912,
+      "logits/rejected": 1.556697130203247,
+      "logps/chosen": -9.863478660583496,
+      "logps/rejected": -11.184213638305664,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.863478660583496,
+      "rewards/margins": 1.3207343816757202,
+      "rewards/rejected": -11.184213638305664,
+      "semantic_entropy": 0.001168654183857143,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 30.692567789415524,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": 1.4296211004257202,
+      "logits/rejected": 1.487903356552124,
+      "logps/chosen": -10.047002792358398,
+      "logps/rejected": -11.22883415222168,
+      "loss": 0.4514,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.047002792358398,
+      "rewards/margins": 1.1818312406539917,
+      "rewards/rejected": -11.22883415222168,
+      "semantic_entropy": 0.001240780926309526,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 27.30552499346248,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": 1.5086898803710938,
+      "logits/rejected": 1.54779052734375,
+      "logps/chosen": -9.717594146728516,
+      "logps/rejected": -10.898451805114746,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -9.717594146728516,
+      "rewards/margins": 1.180857539176941,
+      "rewards/rejected": -10.898451805114746,
+      "semantic_entropy": 0.001399846514686942,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 22.005597008975357,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": 1.5807315111160278,
+      "logits/rejected": 1.635749101638794,
+      "logps/chosen": -9.876286506652832,
+      "logps/rejected": -11.18218994140625,
+      "loss": 0.4577,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.876286506652832,
+      "rewards/margins": 1.3059041500091553,
+      "rewards/rejected": -11.18218994140625,
+      "semantic_entropy": 0.0013822383480146527,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 23.312297686580546,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": 1.4533190727233887,
+      "logits/rejected": 1.4992480278015137,
+      "logps/chosen": -9.817682266235352,
+      "logps/rejected": -11.411906242370605,
+      "loss": 0.3538,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.817682266235352,
+      "rewards/margins": 1.5942233800888062,
+      "rewards/rejected": -11.411906242370605,
+      "semantic_entropy": 0.0013448798563331366,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 31.07472687868582,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": 1.5009876489639282,
+      "logits/rejected": 1.5559322834014893,
+      "logps/chosen": -9.917732238769531,
+      "logps/rejected": -11.508027076721191,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.917732238769531,
+      "rewards/margins": 1.5902947187423706,
+      "rewards/rejected": -11.508027076721191,
+      "semantic_entropy": 0.0011733315186575055,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 23.33223014718689,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": 1.5078582763671875,
+      "logits/rejected": 1.5848573446273804,
+      "logps/chosen": -9.92039680480957,
+      "logps/rejected": -11.50290298461914,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.92039680480957,
+      "rewards/margins": 1.5825055837631226,
+      "rewards/rejected": -11.50290298461914,
+      "semantic_entropy": 0.0011733127757906914,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 24.03130558856795,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": 1.448289155960083,
+      "logits/rejected": 1.4976608753204346,
+      "logps/chosen": -10.024824142456055,
+      "logps/rejected": -11.549975395202637,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.024824142456055,
+      "rewards/margins": 1.525149941444397,
+      "rewards/rejected": -11.549975395202637,
+      "semantic_entropy": 0.0014305777149274945,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 20.316025139512593,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": 1.501543402671814,
+      "logits/rejected": 1.5797150135040283,
+      "logps/chosen": -10.044387817382812,
+      "logps/rejected": -11.517004013061523,
+      "loss": 0.3391,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.044387817382812,
+      "rewards/margins": 1.4726145267486572,
+      "rewards/rejected": -11.517004013061523,
+      "semantic_entropy": 0.0011159274727106094,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 21.58093206768554,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": 1.4724032878875732,
+      "logits/rejected": 1.5720945596694946,
+      "logps/chosen": -9.90136432647705,
+      "logps/rejected": -11.42093563079834,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.90136432647705,
+      "rewards/margins": 1.5195722579956055,
+      "rewards/rejected": -11.42093563079834,
+      "semantic_entropy": 0.002058863639831543,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 24.49164962555193,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": 1.4857685565948486,
+      "logits/rejected": 1.5328562259674072,
+      "logps/chosen": -10.055691719055176,
+      "logps/rejected": -11.517581939697266,
+      "loss": 0.3426,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -10.055691719055176,
+      "rewards/margins": 1.4618914127349854,
+      "rewards/rejected": -11.517581939697266,
+      "semantic_entropy": 0.0010338407009840012,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 26.250503059484732,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": 1.5296485424041748,
+      "logits/rejected": 1.6025310754776,
+      "logps/chosen": -9.907925605773926,
+      "logps/rejected": -11.517618179321289,
+      "loss": 0.364,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.907925605773926,
+      "rewards/margins": 1.6096919775009155,
+      "rewards/rejected": -11.517618179321289,
+      "semantic_entropy": 0.0013157319044694304,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 27.3670753632697,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": 1.544585943222046,
+      "logits/rejected": 1.6338202953338623,
+      "logps/chosen": -10.389750480651855,
+      "logps/rejected": -11.79857063293457,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.389750480651855,
+      "rewards/margins": 1.4088201522827148,
+      "rewards/rejected": -11.79857063293457,
+      "semantic_entropy": 0.0008634948171675205,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 29.35078451237245,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": 1.5191794633865356,
+      "logits/rejected": 1.6608898639678955,
+      "logps/chosen": -10.019041061401367,
+      "logps/rejected": -11.722211837768555,
+      "loss": 0.3332,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.019041061401367,
+      "rewards/margins": 1.7031700611114502,
+      "rewards/rejected": -11.722211837768555,
+      "semantic_entropy": 0.0011815379839390516,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 29.728418607449168,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": 1.4865626096725464,
+      "logits/rejected": 1.5551656484603882,
+      "logps/chosen": -9.884256362915039,
+      "logps/rejected": -11.375465393066406,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.884256362915039,
+      "rewards/margins": 1.4912083148956299,
+      "rewards/rejected": -11.375465393066406,
+      "semantic_entropy": 0.0012135658180341125,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 30.359793245017485,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": 1.514756202697754,
+      "logits/rejected": 1.5471618175506592,
+      "logps/chosen": -9.861509323120117,
+      "logps/rejected": -11.196770668029785,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.861509323120117,
+      "rewards/margins": 1.3352607488632202,
+      "rewards/rejected": -11.196770668029785,
+      "semantic_entropy": 0.0011760194320231676,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 21.846588360200528,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": 1.4426220655441284,
+      "logits/rejected": 1.5418610572814941,
+      "logps/chosen": -9.730645179748535,
+      "logps/rejected": -11.547971725463867,
+      "loss": 0.3038,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -9.730645179748535,
+      "rewards/margins": 1.8173272609710693,
+      "rewards/rejected": -11.547971725463867,
+      "semantic_entropy": 0.0013457763707265258,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 21.295322448842853,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": 1.4526937007904053,
+      "logits/rejected": 1.5168598890304565,
+      "logps/chosen": -9.78083610534668,
+      "logps/rejected": -11.251847267150879,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.78083610534668,
+      "rewards/margins": 1.471010684967041,
+      "rewards/rejected": -11.251847267150879,
+      "semantic_entropy": 0.0012916361447423697,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 20.09117455447824,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": 1.4931894540786743,
+      "logits/rejected": 1.54242742061615,
+      "logps/chosen": -9.945356369018555,
+      "logps/rejected": -11.541784286499023,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.945356369018555,
+      "rewards/margins": 1.5964291095733643,
+      "rewards/rejected": -11.541784286499023,
+      "semantic_entropy": 0.0014251904794946313,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 22.72457740260609,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": 1.4766666889190674,
+      "logits/rejected": 1.5258607864379883,
+      "logps/chosen": -9.796612739562988,
+      "logps/rejected": -11.194389343261719,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.796612739562988,
+      "rewards/margins": 1.3977775573730469,
+      "rewards/rejected": -11.194389343261719,
+      "semantic_entropy": 0.0014370752032846212,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 24.42649924546558,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": 1.5291469097137451,
+      "logits/rejected": 1.589874505996704,
+      "logps/chosen": -10.06102180480957,
+      "logps/rejected": -11.41341781616211,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.06102180480957,
+      "rewards/margins": 1.3523961305618286,
+      "rewards/rejected": -11.41341781616211,
+      "semantic_entropy": 0.0011197816347703338,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 21.83779193667659,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": 1.523011565208435,
+      "logits/rejected": 1.5620019435882568,
+      "logps/chosen": -9.62843132019043,
+      "logps/rejected": -11.091163635253906,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.62843132019043,
+      "rewards/margins": 1.4627327919006348,
+      "rewards/rejected": -11.091163635253906,
+      "semantic_entropy": 0.0015182029455900192,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 24.76731538671959,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": 1.5149961709976196,
+      "logits/rejected": 1.5583561658859253,
+      "logps/chosen": -9.827798843383789,
+      "logps/rejected": -11.360666275024414,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.827798843383789,
+      "rewards/margins": 1.5328668355941772,
+      "rewards/rejected": -11.360666275024414,
+      "semantic_entropy": 0.0014699199236929417,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 20.318294246612506,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": 1.4836949110031128,
+      "logits/rejected": 1.5648882389068604,
+      "logps/chosen": -9.789682388305664,
+      "logps/rejected": -11.377405166625977,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.789682388305664,
+      "rewards/margins": 1.5877234935760498,
+      "rewards/rejected": -11.377405166625977,
+      "semantic_entropy": 0.0013050512643530965,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 24.702666398620813,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": 1.5105592012405396,
+      "logits/rejected": 1.56656813621521,
+      "logps/chosen": -9.999165534973145,
+      "logps/rejected": -11.639673233032227,
+      "loss": 0.3597,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.999165534973145,
+      "rewards/margins": 1.6405086517333984,
+      "rewards/rejected": -11.639673233032227,
+      "semantic_entropy": 0.0009913721587508917,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 32.031936376173924,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": 1.549480676651001,
+      "logits/rejected": 1.5792101621627808,
+      "logps/chosen": -10.011869430541992,
+      "logps/rejected": -11.583820343017578,
+      "loss": 0.3674,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.011869430541992,
+      "rewards/margins": 1.571951150894165,
+      "rewards/rejected": -11.583820343017578,
+      "semantic_entropy": 0.0014432771131396294,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 32.23195037632725,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": 1.501851201057434,
+      "logits/rejected": 1.5488709211349487,
+      "logps/chosen": -10.081354141235352,
+      "logps/rejected": -11.388985633850098,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.081354141235352,
+      "rewards/margins": 1.307631254196167,
+      "rewards/rejected": -11.388985633850098,
+      "semantic_entropy": 0.0009836169192567468,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 23.277869742644675,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": 1.502726674079895,
+      "logits/rejected": 1.5708518028259277,
+      "logps/chosen": -10.007516860961914,
+      "logps/rejected": -11.511489868164062,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.007516860961914,
+      "rewards/margins": 1.503974199295044,
+      "rewards/rejected": -11.511489868164062,
+      "semantic_entropy": 0.0019413683330640197,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 23.14214483808196,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": 1.438901662826538,
+      "logits/rejected": 1.5130857229232788,
+      "logps/chosen": -9.874427795410156,
+      "logps/rejected": -11.526244163513184,
+      "loss": 0.3477,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.874427795410156,
+      "rewards/margins": 1.6518173217773438,
+      "rewards/rejected": -11.526244163513184,
+      "semantic_entropy": 0.0013633475173264742,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 24.822291176175135,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": 1.5477114915847778,
+      "logits/rejected": 1.589941382408142,
+      "logps/chosen": -9.844869613647461,
+      "logps/rejected": -11.410746574401855,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.844869613647461,
+      "rewards/margins": 1.5658771991729736,
+      "rewards/rejected": -11.410746574401855,
+      "semantic_entropy": 0.001193707576021552,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 25.989449384656428,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": 1.4273157119750977,
+      "logits/rejected": 1.4953744411468506,
+      "logps/chosen": -9.773181915283203,
+      "logps/rejected": -11.167338371276855,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.773181915283203,
+      "rewards/margins": 1.3941569328308105,
+      "rewards/rejected": -11.167338371276855,
+      "semantic_entropy": 0.0013838001759722829,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 28.405835696562807,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": 1.4986196756362915,
+      "logits/rejected": 1.4894325733184814,
+      "logps/chosen": -10.018640518188477,
+      "logps/rejected": -11.301322937011719,
+      "loss": 0.4357,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.018640518188477,
+      "rewards/margins": 1.2826826572418213,
+      "rewards/rejected": -11.301322937011719,
+      "semantic_entropy": 0.0013308317866176367,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 26.90930750431657,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": 1.485512614250183,
+      "logits/rejected": 1.5612857341766357,
+      "logps/chosen": -9.922945976257324,
+      "logps/rejected": -11.33785629272461,
+      "loss": 0.3973,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.922945976257324,
+      "rewards/margins": 1.4149110317230225,
+      "rewards/rejected": -11.33785629272461,
+      "semantic_entropy": 0.001141394255682826,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 18.423371742037688,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": 1.488837480545044,
+      "logits/rejected": 1.5619179010391235,
+      "logps/chosen": -9.876794815063477,
+      "logps/rejected": -11.68836498260498,
+      "loss": 0.3278,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -9.876794815063477,
+      "rewards/margins": 1.8115713596343994,
+      "rewards/rejected": -11.68836498260498,
+      "semantic_entropy": 0.0012125041102990508,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 23.741946820292664,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": 1.4406496286392212,
+      "logits/rejected": 1.5235109329223633,
+      "logps/chosen": -9.995210647583008,
+      "logps/rejected": -11.549077033996582,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.995210647583008,
+      "rewards/margins": 1.553865671157837,
+      "rewards/rejected": -11.549077033996582,
+      "semantic_entropy": 0.0012844514567404985,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 18.15651267027521,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": 1.4592258930206299,
+      "logits/rejected": 1.534920334815979,
+      "logps/chosen": -9.843841552734375,
+      "logps/rejected": -11.507162094116211,
+      "loss": 0.3493,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.843841552734375,
+      "rewards/margins": 1.6633214950561523,
+      "rewards/rejected": -11.507162094116211,
+      "semantic_entropy": 0.0014445041306316853,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 26.81111441309598,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": 1.4276403188705444,
+      "logits/rejected": 1.507490634918213,
+      "logps/chosen": -9.746679306030273,
+      "logps/rejected": -11.210615158081055,
+      "loss": 0.4115,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.746679306030273,
+      "rewards/margins": 1.4639360904693604,
+      "rewards/rejected": -11.210615158081055,
+      "semantic_entropy": 0.001573445275425911,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 27.37027800544906,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": 1.4447853565216064,
+      "logits/rejected": 1.4735043048858643,
+      "logps/chosen": -9.800973892211914,
+      "logps/rejected": -11.194499969482422,
+      "loss": 0.4324,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.800973892211914,
+      "rewards/margins": 1.3935270309448242,
+      "rewards/rejected": -11.194499969482422,
+      "semantic_entropy": 0.0011505426373332739,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 21.73091914316813,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": 1.4470137357711792,
+      "logits/rejected": 1.5226787328720093,
+      "logps/chosen": -9.82433795928955,
+      "logps/rejected": -11.227839469909668,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.82433795928955,
+      "rewards/margins": 1.4035013914108276,
+      "rewards/rejected": -11.227839469909668,
+      "semantic_entropy": 0.0013543912209570408,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 21.728259765742948,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": 1.426914930343628,
+      "logits/rejected": 1.4892642498016357,
+      "logps/chosen": -9.477563858032227,
+      "logps/rejected": -11.032739639282227,
+      "loss": 0.3716,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.477563858032227,
+      "rewards/margins": 1.5551749467849731,
+      "rewards/rejected": -11.032739639282227,
+      "semantic_entropy": 0.0014651247765868902,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 19.136343426423494,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": 1.411186933517456,
+      "logits/rejected": 1.4711333513259888,
+      "logps/chosen": -9.672426223754883,
+      "logps/rejected": -11.115825653076172,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.672426223754883,
+      "rewards/margins": 1.44339919090271,
+      "rewards/rejected": -11.115825653076172,
+      "semantic_entropy": 0.0016790505032986403,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 22.31988215907516,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": 1.376347303390503,
+      "logits/rejected": 1.465395212173462,
+      "logps/chosen": -9.704439163208008,
+      "logps/rejected": -11.212620735168457,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.704439163208008,
+      "rewards/margins": 1.5081816911697388,
+      "rewards/rejected": -11.212620735168457,
+      "semantic_entropy": 0.0013334506656974554,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 20.5165736743471,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": 1.444379210472107,
+      "logits/rejected": 1.5189433097839355,
+      "logps/chosen": -9.829694747924805,
+      "logps/rejected": -11.526727676391602,
+      "loss": 0.3438,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.829694747924805,
+      "rewards/margins": 1.6970336437225342,
+      "rewards/rejected": -11.526727676391602,
+      "semantic_entropy": 0.0014892593026161194,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 29.146897935456025,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": 1.4839118719100952,
+      "logits/rejected": 1.5411696434020996,
+      "logps/chosen": -9.803966522216797,
+      "logps/rejected": -11.178397178649902,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -9.803966522216797,
+      "rewards/margins": 1.37443208694458,
+      "rewards/rejected": -11.178397178649902,
+      "semantic_entropy": 0.0014999612467363477,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 16.384449038463472,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": 1.4622882604599,
+      "logits/rejected": 1.5354633331298828,
+      "logps/chosen": -9.846537590026855,
+      "logps/rejected": -11.50123119354248,
+      "loss": 0.3611,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -9.846537590026855,
+      "rewards/margins": 1.6546930074691772,
+      "rewards/rejected": -11.50123119354248,
+      "semantic_entropy": 0.0015144520439207554,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 24.141806707700095,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": 1.4828425645828247,
+      "logits/rejected": 1.5580703020095825,
+      "logps/chosen": -9.904866218566895,
+      "logps/rejected": -11.385355949401855,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.904866218566895,
+      "rewards/margins": 1.4804903268814087,
+      "rewards/rejected": -11.385355949401855,
+      "semantic_entropy": 0.0011582615552470088,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 28.911749349580177,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": 1.4710850715637207,
+      "logits/rejected": 1.506476640701294,
+      "logps/chosen": -9.883294105529785,
+      "logps/rejected": -11.245233535766602,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.883294105529785,
+      "rewards/margins": 1.3619390726089478,
+      "rewards/rejected": -11.245233535766602,
+      "semantic_entropy": 0.0012728290166705847,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 29.754120497165967,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": 1.4484373331069946,
+      "logits/rejected": 1.4959962368011475,
+      "logps/chosen": -9.761709213256836,
+      "logps/rejected": -11.375018119812012,
+      "loss": 0.3618,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.761709213256836,
+      "rewards/margins": 1.6133092641830444,
+      "rewards/rejected": -11.375018119812012,
+      "semantic_entropy": 0.0014710075920447707,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 32.848478684848466,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": 1.4768455028533936,
+      "logits/rejected": 1.5111054182052612,
+      "logps/chosen": -9.85079288482666,
+      "logps/rejected": -11.475672721862793,
+      "loss": 0.3346,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -9.85079288482666,
+      "rewards/margins": 1.6248810291290283,
+      "rewards/rejected": -11.475672721862793,
+      "semantic_entropy": 0.0013558734208345413,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 35.40406871175006,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": 1.440197467803955,
+      "logits/rejected": 1.4964927434921265,
+      "logps/chosen": -9.999314308166504,
+      "logps/rejected": -11.29109001159668,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.999314308166504,
+      "rewards/margins": 1.2917758226394653,
+      "rewards/rejected": -11.29109001159668,
+      "semantic_entropy": 0.000971008266787976,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 24.361032890721205,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": 1.4559297561645508,
+      "logits/rejected": 1.4832683801651,
+      "logps/chosen": -9.892183303833008,
+      "logps/rejected": -11.146453857421875,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.892183303833008,
+      "rewards/margins": 1.2542718648910522,
+      "rewards/rejected": -11.146453857421875,
+      "semantic_entropy": 0.001106268959119916,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 21.246808700693382,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": 1.4496301412582397,
+      "logits/rejected": 1.5408145189285278,
+      "logps/chosen": -9.648874282836914,
+      "logps/rejected": -10.991243362426758,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.648874282836914,
+      "rewards/margins": 1.342370629310608,
+      "rewards/rejected": -10.991243362426758,
+      "semantic_entropy": 0.0015067103086039424,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 26.568115867866393,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": 1.4180233478546143,
+      "logits/rejected": 1.4866046905517578,
+      "logps/chosen": -9.557000160217285,
+      "logps/rejected": -11.10246753692627,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.557000160217285,
+      "rewards/margins": 1.545467495918274,
+      "rewards/rejected": -11.10246753692627,
+      "semantic_entropy": 0.0021724903490394354,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 26.122069612929767,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": 1.4730074405670166,
+      "logits/rejected": 1.5412195920944214,
+      "logps/chosen": -9.722512245178223,
+      "logps/rejected": -11.505258560180664,
+      "loss": 0.3133,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -9.722512245178223,
+      "rewards/margins": 1.7827463150024414,
+      "rewards/rejected": -11.505258560180664,
+      "semantic_entropy": 0.0015057773562148213,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 31.896495455672632,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": 1.4230425357818604,
+      "logits/rejected": 1.5042498111724854,
+      "logps/chosen": -9.838078498840332,
+      "logps/rejected": -11.266889572143555,
+      "loss": 0.4493,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -9.838078498840332,
+      "rewards/margins": 1.428810954093933,
+      "rewards/rejected": -11.266889572143555,
+      "semantic_entropy": 0.0013263777364045382,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 19.877216399052514,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": 1.4550297260284424,
+      "logits/rejected": 1.509216070175171,
+      "logps/chosen": -9.762828826904297,
+      "logps/rejected": -11.312752723693848,
+      "loss": 0.3834,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.762828826904297,
+      "rewards/margins": 1.5499237775802612,
+      "rewards/rejected": -11.312752723693848,
+      "semantic_entropy": 0.001766704022884369,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 28.44555447235624,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": 1.4615468978881836,
+      "logits/rejected": 1.5218226909637451,
+      "logps/chosen": -9.743903160095215,
+      "logps/rejected": -11.328373908996582,
+      "loss": 0.4038,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.743903160095215,
+      "rewards/margins": 1.5844703912734985,
+      "rewards/rejected": -11.328373908996582,
+      "semantic_entropy": 0.001476147910580039,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 24.481566137363576,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": 1.4401724338531494,
+      "logits/rejected": 1.5299865007400513,
+      "logps/chosen": -9.784215927124023,
+      "logps/rejected": -11.197373390197754,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.784215927124023,
+      "rewards/margins": 1.4131560325622559,
+      "rewards/rejected": -11.197373390197754,
+      "semantic_entropy": 0.0012915116967633367,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 25.546828033929998,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": 1.4347535371780396,
+      "logits/rejected": 1.5100210905075073,
+      "logps/chosen": -9.743106842041016,
+      "logps/rejected": -11.27607536315918,
+      "loss": 0.3682,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.743106842041016,
+      "rewards/margins": 1.532968521118164,
+      "rewards/rejected": -11.27607536315918,
+      "semantic_entropy": 0.001214087475091219,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 22.98060250859113,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": 1.4919458627700806,
+      "logits/rejected": 1.5697729587554932,
+      "logps/chosen": -9.8408784866333,
+      "logps/rejected": -11.456284523010254,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.8408784866333,
+      "rewards/margins": 1.6154060363769531,
+      "rewards/rejected": -11.456284523010254,
+      "semantic_entropy": 0.0013280571438372135,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 29.452630621271066,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": 1.4342677593231201,
+      "logits/rejected": 1.5142438411712646,
+      "logps/chosen": -9.878989219665527,
+      "logps/rejected": -11.312902450561523,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.878989219665527,
+      "rewards/margins": 1.433912754058838,
+      "rewards/rejected": -11.312902450561523,
+      "semantic_entropy": 0.0012886624317616224,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 28.416571323940417,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": 1.457890272140503,
+      "logits/rejected": 1.5367685556411743,
+      "logps/chosen": -9.952252388000488,
+      "logps/rejected": -11.561394691467285,
+      "loss": 0.3661,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.952252388000488,
+      "rewards/margins": 1.6091426610946655,
+      "rewards/rejected": -11.561394691467285,
+      "semantic_entropy": 0.0015111168613657355,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 24.831064217969097,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": 1.4750638008117676,
+      "logits/rejected": 1.5263314247131348,
+      "logps/chosen": -9.90894603729248,
+      "logps/rejected": -11.369284629821777,
+      "loss": 0.4046,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.90894603729248,
+      "rewards/margins": 1.4603372812271118,
+      "rewards/rejected": -11.369284629821777,
+      "semantic_entropy": 0.0013143583200871944,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 25.86102356797801,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": 1.4642655849456787,
+      "logits/rejected": 1.5473498106002808,
+      "logps/chosen": -9.955537796020508,
+      "logps/rejected": -11.503901481628418,
+      "loss": 0.3799,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.955537796020508,
+      "rewards/margins": 1.5483639240264893,
+      "rewards/rejected": -11.503901481628418,
+      "semantic_entropy": 0.0013076381292194128,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 26.310887996237142,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": 1.5393015146255493,
+      "logits/rejected": 1.5615898370742798,
+      "logps/chosen": -10.098685264587402,
+      "logps/rejected": -11.455918312072754,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.098685264587402,
+      "rewards/margins": 1.357232928276062,
+      "rewards/rejected": -11.455918312072754,
+      "semantic_entropy": 0.0013254726072773337,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 26.003707842936443,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": 1.5482295751571655,
+      "logits/rejected": 1.5943553447723389,
+      "logps/chosen": -9.896836280822754,
+      "logps/rejected": -11.692975044250488,
+      "loss": 0.3182,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.896836280822754,
+      "rewards/margins": 1.7961393594741821,
+      "rewards/rejected": -11.692975044250488,
+      "semantic_entropy": 0.0014097983948886395,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 29.177810057323793,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": 1.4424830675125122,
+      "logits/rejected": 1.5542315244674683,
+      "logps/chosen": -9.83538818359375,
+      "logps/rejected": -11.595006942749023,
+      "loss": 0.3496,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.83538818359375,
+      "rewards/margins": 1.7596184015274048,
+      "rewards/rejected": -11.595006942749023,
+      "semantic_entropy": 0.0013609540183097124,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 24.55124392896399,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": 1.499820590019226,
+      "logits/rejected": 1.591408610343933,
+      "logps/chosen": -9.836517333984375,
+      "logps/rejected": -11.408825874328613,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.836517333984375,
+      "rewards/margins": 1.572308897972107,
+      "rewards/rejected": -11.408825874328613,
+      "semantic_entropy": 0.0014409322757273912,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 30.83793404932353,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": 1.3904567956924438,
+      "logits/rejected": 1.463687539100647,
+      "logps/chosen": -10.03003978729248,
+      "logps/rejected": -11.293846130371094,
+      "loss": 0.435,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.03003978729248,
+      "rewards/margins": 1.2638064622879028,
+      "rewards/rejected": -11.293846130371094,
+      "semantic_entropy": 0.0009674840839579701,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 19.38057342828396,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": 1.5031569004058838,
+      "logits/rejected": 1.587974190711975,
+      "logps/chosen": -9.837369918823242,
+      "logps/rejected": -11.363439559936523,
+      "loss": 0.358,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -9.837369918823242,
+      "rewards/margins": 1.5260690450668335,
+      "rewards/rejected": -11.363439559936523,
+      "semantic_entropy": 0.001660201116465032,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 28.949449763911982,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": 1.531747579574585,
+      "logits/rejected": 1.5994060039520264,
+      "logps/chosen": -9.91172981262207,
+      "logps/rejected": -11.480871200561523,
+      "loss": 0.393,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.91172981262207,
+      "rewards/margins": 1.5691423416137695,
+      "rewards/rejected": -11.480871200561523,
+      "semantic_entropy": 0.0012467264896258712,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 23.730424463883175,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": 1.4901070594787598,
+      "logits/rejected": 1.5573089122772217,
+      "logps/chosen": -9.990477561950684,
+      "logps/rejected": -11.282828330993652,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -9.990477561950684,
+      "rewards/margins": 1.2923511266708374,
+      "rewards/rejected": -11.282828330993652,
+      "semantic_entropy": 0.0012343160342425108,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": 1.5684434175491333,
+      "eval_logits/rejected": 1.613440990447998,
+      "eval_logps/chosen": -10.11075496673584,
+      "eval_logps/rejected": -11.300090789794922,
+      "eval_loss": 0.5428682565689087,
+      "eval_rewards/accuracies": 0.7218101024627686,
+      "eval_rewards/chosen": -10.11075496673584,
+      "eval_rewards/margins": 1.189336895942688,
+      "eval_rewards/rejected": -11.300090789794922,
+      "eval_runtime": 34.9258,
+      "eval_samples_per_second": 38.51,
+      "eval_semantic_entropy": 0.0012318164808675647,
+      "eval_steps_per_second": 9.649,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 26.80446243731935,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": 1.4368853569030762,
+      "logits/rejected": 1.4889434576034546,
+      "logps/chosen": -9.835697174072266,
+      "logps/rejected": -11.348416328430176,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.835697174072266,
+      "rewards/margins": 1.512718915939331,
+      "rewards/rejected": -11.348416328430176,
+      "semantic_entropy": 0.0012520228046923876,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 20.727196333855154,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": 1.517411470413208,
+      "logits/rejected": 1.6118834018707275,
+      "logps/chosen": -10.15757942199707,
+      "logps/rejected": -11.814286231994629,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.15757942199707,
+      "rewards/margins": 1.656707525253296,
+      "rewards/rejected": -11.814286231994629,
+      "semantic_entropy": 0.0011890430469065905,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 23.570023871163162,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": 1.5071061849594116,
+      "logits/rejected": 1.5814752578735352,
+      "logps/chosen": -10.038030624389648,
+      "logps/rejected": -11.535477638244629,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.038030624389648,
+      "rewards/margins": 1.4974470138549805,
+      "rewards/rejected": -11.535477638244629,
+      "semantic_entropy": 0.0010292748920619488,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 26.825024397713104,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": 1.4775416851043701,
+      "logits/rejected": 1.5152549743652344,
+      "logps/chosen": -9.90471363067627,
+      "logps/rejected": -11.27821159362793,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.90471363067627,
+      "rewards/margins": 1.3734980821609497,
+      "rewards/rejected": -11.27821159362793,
+      "semantic_entropy": 0.0011284246575087309,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 20.548950577603808,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": 1.4935557842254639,
+      "logits/rejected": 1.5690988302230835,
+      "logps/chosen": -9.81724739074707,
+      "logps/rejected": -11.305159568786621,
+      "loss": 0.3323,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -9.81724739074707,
+      "rewards/margins": 1.4879122972488403,
+      "rewards/rejected": -11.305159568786621,
+      "semantic_entropy": 0.0013500151690095663,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 28.707593654998725,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": 1.5050101280212402,
+      "logits/rejected": 1.5738166570663452,
+      "logps/chosen": -9.983817100524902,
+      "logps/rejected": -11.625394821166992,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.983817100524902,
+      "rewards/margins": 1.6415760517120361,
+      "rewards/rejected": -11.625394821166992,
+      "semantic_entropy": 0.0010958422208204865,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 18.82626933199735,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": 1.5082008838653564,
+      "logits/rejected": 1.5642175674438477,
+      "logps/chosen": -10.123435974121094,
+      "logps/rejected": -11.648358345031738,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.123435974121094,
+      "rewards/margins": 1.5249226093292236,
+      "rewards/rejected": -11.648358345031738,
+      "semantic_entropy": 0.001055243075825274,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 24.175091114616595,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": 1.421632170677185,
+      "logits/rejected": 1.5455900430679321,
+      "logps/chosen": -9.930368423461914,
+      "logps/rejected": -11.423591613769531,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.930368423461914,
+      "rewards/margins": 1.4932228326797485,
+      "rewards/rejected": -11.423591613769531,
+      "semantic_entropy": 0.0011863454710692167,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 29.32225435821126,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": 1.483758568763733,
+      "logits/rejected": 1.5719951391220093,
+      "logps/chosen": -10.022625923156738,
+      "logps/rejected": -11.36660385131836,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.022625923156738,
+      "rewards/margins": 1.343980073928833,
+      "rewards/rejected": -11.36660385131836,
+      "semantic_entropy": 0.001039310940541327,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 31.74021133471401,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": 1.4802435636520386,
+      "logits/rejected": 1.5522558689117432,
+      "logps/chosen": -10.092161178588867,
+      "logps/rejected": -11.669079780578613,
+      "loss": 0.3856,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.092161178588867,
+      "rewards/margins": 1.5769187211990356,
+      "rewards/rejected": -11.669079780578613,
+      "semantic_entropy": 0.0010815097484737635,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 23.525390272407446,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": 1.4356894493103027,
+      "logits/rejected": 1.5143753290176392,
+      "logps/chosen": -9.839639663696289,
+      "logps/rejected": -11.42857551574707,
+      "loss": 0.393,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.839639663696289,
+      "rewards/margins": 1.5889370441436768,
+      "rewards/rejected": -11.42857551574707,
+      "semantic_entropy": 0.0012518191942945123,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 23.181029131780956,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": 1.4897650480270386,
+      "logits/rejected": 1.5745751857757568,
+      "logps/chosen": -10.037358283996582,
+      "logps/rejected": -11.436424255371094,
+      "loss": 0.4163,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.037358283996582,
+      "rewards/margins": 1.3990662097930908,
+      "rewards/rejected": -11.436424255371094,
+      "semantic_entropy": 0.001065070740878582,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 22.062151293192912,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": 1.495430827140808,
+      "logits/rejected": 1.6148923635482788,
+      "logps/chosen": -10.149862289428711,
+      "logps/rejected": -11.68945026397705,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.149862289428711,
+      "rewards/margins": 1.5395877361297607,
+      "rewards/rejected": -11.68945026397705,
+      "semantic_entropy": 0.001382522052153945,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 19.543732383786132,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": 1.5535796880722046,
+      "logits/rejected": 1.6355855464935303,
+      "logps/chosen": -10.075155258178711,
+      "logps/rejected": -11.673447608947754,
+      "loss": 0.39,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.075155258178711,
+      "rewards/margins": 1.5982916355133057,
+      "rewards/rejected": -11.673447608947754,
+      "semantic_entropy": 0.001119624706916511,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 22.056365616091757,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": 1.5047376155853271,
+      "logits/rejected": 1.6005674600601196,
+      "logps/chosen": -10.19676399230957,
+      "logps/rejected": -11.564226150512695,
+      "loss": 0.4078,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.19676399230957,
+      "rewards/margins": 1.3674620389938354,
+      "rewards/rejected": -11.564226150512695,
+      "semantic_entropy": 0.0010559256188571453,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 34.39345795377231,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": 1.5413225889205933,
+      "logits/rejected": 1.6162490844726562,
+      "logps/chosen": -10.258383750915527,
+      "logps/rejected": -11.624505996704102,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.258383750915527,
+      "rewards/margins": 1.3661216497421265,
+      "rewards/rejected": -11.624505996704102,
+      "semantic_entropy": 0.0008789139683358371,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 29.681722966379812,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": 1.52113676071167,
+      "logits/rejected": 1.5714824199676514,
+      "logps/chosen": -10.284153938293457,
+      "logps/rejected": -11.972073554992676,
+      "loss": 0.3345,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.284153938293457,
+      "rewards/margins": 1.6879189014434814,
+      "rewards/rejected": -11.972073554992676,
+      "semantic_entropy": 0.0010614495258778334,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 22.2313460160698,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": 1.5357773303985596,
+      "logits/rejected": 1.6095386743545532,
+      "logps/chosen": -10.444416046142578,
+      "logps/rejected": -11.969429969787598,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.444416046142578,
+      "rewards/margins": 1.5250166654586792,
+      "rewards/rejected": -11.969429969787598,
+      "semantic_entropy": 0.0008121196296997368,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 23.141266507824003,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": 1.459090232849121,
+      "logits/rejected": 1.5638803243637085,
+      "logps/chosen": -10.199346542358398,
+      "logps/rejected": -11.487680435180664,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.199346542358398,
+      "rewards/margins": 1.2883353233337402,
+      "rewards/rejected": -11.487680435180664,
+      "semantic_entropy": 0.0010306842159479856,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 26.40343899474739,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": 1.5772058963775635,
+      "logits/rejected": 1.6005117893218994,
+      "logps/chosen": -10.254804611206055,
+      "logps/rejected": -11.61518669128418,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.254804611206055,
+      "rewards/margins": 1.3603813648223877,
+      "rewards/rejected": -11.61518669128418,
+      "semantic_entropy": 0.0009136631269939244,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 26.31787586440611,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": 1.572540521621704,
+      "logits/rejected": 1.6684963703155518,
+      "logps/chosen": -10.281415939331055,
+      "logps/rejected": -11.670991897583008,
+      "loss": 0.4214,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.281415939331055,
+      "rewards/margins": 1.3895772695541382,
+      "rewards/rejected": -11.670991897583008,
+      "semantic_entropy": 0.0008412927272729576,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 28.00143915068069,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": 1.5381906032562256,
+      "logits/rejected": 1.5870187282562256,
+      "logps/chosen": -10.216726303100586,
+      "logps/rejected": -11.549688339233398,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -10.216726303100586,
+      "rewards/margins": 1.332960844039917,
+      "rewards/rejected": -11.549688339233398,
+      "semantic_entropy": 0.00095493218395859,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 20.849558122705535,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": 1.5028278827667236,
+      "logits/rejected": 1.5328959226608276,
+      "logps/chosen": -10.06006908416748,
+      "logps/rejected": -11.49559211730957,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.06006908416748,
+      "rewards/margins": 1.4355219602584839,
+      "rewards/rejected": -11.49559211730957,
+      "semantic_entropy": 0.001159831415861845,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 23.616946510684134,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": 1.5664379596710205,
+      "logits/rejected": 1.607773780822754,
+      "logps/chosen": -10.048799514770508,
+      "logps/rejected": -11.665739059448242,
+      "loss": 0.3625,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.048799514770508,
+      "rewards/margins": 1.6169402599334717,
+      "rewards/rejected": -11.665739059448242,
+      "semantic_entropy": 0.0012066186172887683,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 39.89622796621888,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": 1.5043424367904663,
+      "logits/rejected": 1.6022990942001343,
+      "logps/chosen": -10.191396713256836,
+      "logps/rejected": -11.776432037353516,
+      "loss": 0.389,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.191396713256836,
+      "rewards/margins": 1.5850341320037842,
+      "rewards/rejected": -11.776432037353516,
+      "semantic_entropy": 0.0010599000379443169,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 25.71907166103573,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": 1.5004667043685913,
+      "logits/rejected": 1.5573337078094482,
+      "logps/chosen": -10.138898849487305,
+      "logps/rejected": -11.470991134643555,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.138898849487305,
+      "rewards/margins": 1.332092523574829,
+      "rewards/rejected": -11.470991134643555,
+      "semantic_entropy": 0.001023524790070951,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 32.15187578594313,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": 1.6041576862335205,
+      "logits/rejected": 1.6338592767715454,
+      "logps/chosen": -10.078697204589844,
+      "logps/rejected": -11.458585739135742,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.078697204589844,
+      "rewards/margins": 1.3798892498016357,
+      "rewards/rejected": -11.458585739135742,
+      "semantic_entropy": 0.0010076850885525346,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 25.62309093947806,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": 1.5457028150558472,
+      "logits/rejected": 1.6231114864349365,
+      "logps/chosen": -10.07374095916748,
+      "logps/rejected": -11.358223915100098,
+      "loss": 0.4696,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -10.07374095916748,
+      "rewards/margins": 1.284482717514038,
+      "rewards/rejected": -11.358223915100098,
+      "semantic_entropy": 0.001234628725796938,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 30.95070219268801,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": 1.568800926208496,
+      "logits/rejected": 1.6359277963638306,
+      "logps/chosen": -10.056288719177246,
+      "logps/rejected": -11.441617965698242,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.056288719177246,
+      "rewards/margins": 1.3853291273117065,
+      "rewards/rejected": -11.441617965698242,
+      "semantic_entropy": 0.0010367499198764563,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 26.765584773040764,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": 1.5149106979370117,
+      "logits/rejected": 1.5873472690582275,
+      "logps/chosen": -9.909212112426758,
+      "logps/rejected": -11.47928524017334,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -9.909212112426758,
+      "rewards/margins": 1.5700721740722656,
+      "rewards/rejected": -11.47928524017334,
+      "semantic_entropy": 0.0011117797112092376,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 30.456974239158672,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": 1.4962522983551025,
+      "logits/rejected": 1.5772616863250732,
+      "logps/chosen": -10.126401901245117,
+      "logps/rejected": -11.27337646484375,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -10.126401901245117,
+      "rewards/margins": 1.1469743251800537,
+      "rewards/rejected": -11.27337646484375,
+      "semantic_entropy": 0.0009734302875585854,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 32.646456141218465,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": 1.493533730506897,
+      "logits/rejected": 1.5498325824737549,
+      "logps/chosen": -10.09638786315918,
+      "logps/rejected": -11.673288345336914,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.09638786315918,
+      "rewards/margins": 1.5769010782241821,
+      "rewards/rejected": -11.673288345336914,
+      "semantic_entropy": 0.001349339378066361,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 25.48492799272598,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": 1.4421873092651367,
+      "logits/rejected": 1.5155887603759766,
+      "logps/chosen": -9.876604080200195,
+      "logps/rejected": -11.478919982910156,
+      "loss": 0.3601,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -9.876604080200195,
+      "rewards/margins": 1.602315902709961,
+      "rewards/rejected": -11.478919982910156,
+      "semantic_entropy": 0.0013047512620687485,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 22.2368948860047,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": 1.5957733392715454,
+      "logits/rejected": 1.6580222845077515,
+      "logps/chosen": -10.192060470581055,
+      "logps/rejected": -11.561388969421387,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -10.192060470581055,
+      "rewards/margins": 1.3693287372589111,
+      "rewards/rejected": -11.561388969421387,
+      "semantic_entropy": 0.0011783704394474626,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 32.828170602586944,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": 1.5343420505523682,
+      "logits/rejected": 1.5872782468795776,
+      "logps/chosen": -9.967016220092773,
+      "logps/rejected": -11.513630867004395,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.967016220092773,
+      "rewards/margins": 1.546614408493042,
+      "rewards/rejected": -11.513630867004395,
+      "semantic_entropy": 0.0011824017856270075,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 22.150925478110018,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": 1.5033422708511353,
+      "logits/rejected": 1.5440149307250977,
+      "logps/chosen": -9.912618637084961,
+      "logps/rejected": -11.251727104187012,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.912618637084961,
+      "rewards/margins": 1.3391094207763672,
+      "rewards/rejected": -11.251727104187012,
+      "semantic_entropy": 0.0012149253161624074,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 21.91901840116828,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": 1.51212477684021,
+      "logits/rejected": 1.6049045324325562,
+      "logps/chosen": -10.077945709228516,
+      "logps/rejected": -11.584087371826172,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.077945709228516,
+      "rewards/margins": 1.506142020225525,
+      "rewards/rejected": -11.584087371826172,
+      "semantic_entropy": 0.001076513435691595,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 22.60660668382491,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": 1.5202964544296265,
+      "logits/rejected": 1.591288685798645,
+      "logps/chosen": -10.011811256408691,
+      "logps/rejected": -11.426106452941895,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.011811256408691,
+      "rewards/margins": 1.4142953157424927,
+      "rewards/rejected": -11.426106452941895,
+      "semantic_entropy": 0.0010625595459714532,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 23.18815470134787,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": 1.5327122211456299,
+      "logits/rejected": 1.606030821800232,
+      "logps/chosen": -10.038012504577637,
+      "logps/rejected": -11.574644088745117,
+      "loss": 0.3937,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.038012504577637,
+      "rewards/margins": 1.5366312265396118,
+      "rewards/rejected": -11.574644088745117,
+      "semantic_entropy": 0.0010489935521036386,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 24.57137941288666,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": 1.5659981966018677,
+      "logits/rejected": 1.6638071537017822,
+      "logps/chosen": -10.336030960083008,
+      "logps/rejected": -11.911476135253906,
+      "loss": 0.3493,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.336030960083008,
+      "rewards/margins": 1.5754458904266357,
+      "rewards/rejected": -11.911476135253906,
+      "semantic_entropy": 0.000934068113565445,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 23.671208652951993,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": 1.548696756362915,
+      "logits/rejected": 1.6199719905853271,
+      "logps/chosen": -10.271745681762695,
+      "logps/rejected": -11.783834457397461,
+      "loss": 0.376,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.271745681762695,
+      "rewards/margins": 1.5120904445648193,
+      "rewards/rejected": -11.783834457397461,
+      "semantic_entropy": 0.0008010541787371039,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 26.96661466293178,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": 1.5522277355194092,
+      "logits/rejected": 1.6010116338729858,
+      "logps/chosen": -10.277697563171387,
+      "logps/rejected": -11.712736129760742,
+      "loss": 0.436,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.277697563171387,
+      "rewards/margins": 1.4350366592407227,
+      "rewards/rejected": -11.712736129760742,
+      "semantic_entropy": 0.0009325292194262147,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 24.930263676526543,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": 1.5722416639328003,
+      "logits/rejected": 1.6692075729370117,
+      "logps/chosen": -10.248922348022461,
+      "logps/rejected": -11.794528007507324,
+      "loss": 0.3763,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.248922348022461,
+      "rewards/margins": 1.5456058979034424,
+      "rewards/rejected": -11.794528007507324,
+      "semantic_entropy": 0.0009640323696658015,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 20.73620386729852,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": 1.5762066841125488,
+      "logits/rejected": 1.6774342060089111,
+      "logps/chosen": -10.049752235412598,
+      "logps/rejected": -11.622278213500977,
+      "loss": 0.342,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.049752235412598,
+      "rewards/margins": 1.572526216506958,
+      "rewards/rejected": -11.622278213500977,
+      "semantic_entropy": 0.000972963694948703,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 22.529231088703263,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": 1.5449802875518799,
+      "logits/rejected": 1.5576965808868408,
+      "logps/chosen": -10.125862121582031,
+      "logps/rejected": -11.700109481811523,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.125862121582031,
+      "rewards/margins": 1.5742456912994385,
+      "rewards/rejected": -11.700109481811523,
+      "semantic_entropy": 0.001002090866677463,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 32.041012367850506,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": 1.513304591178894,
+      "logits/rejected": 1.567341685295105,
+      "logps/chosen": -9.958353042602539,
+      "logps/rejected": -11.51988410949707,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.958353042602539,
+      "rewards/margins": 1.561531662940979,
+      "rewards/rejected": -11.51988410949707,
+      "semantic_entropy": 0.0011508489260450006,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 34.523234684218494,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": 1.6713483333587646,
+      "logits/rejected": 1.7295879125595093,
+      "logps/chosen": -10.327237129211426,
+      "logps/rejected": -11.88557243347168,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.327237129211426,
+      "rewards/margins": 1.5583356618881226,
+      "rewards/rejected": -11.88557243347168,
+      "semantic_entropy": 0.0007959214271977544,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 21.279955348419136,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": 1.5427848100662231,
+      "logits/rejected": 1.6194992065429688,
+      "logps/chosen": -10.082921028137207,
+      "logps/rejected": -11.501545906066895,
+      "loss": 0.3774,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.082921028137207,
+      "rewards/margins": 1.4186251163482666,
+      "rewards/rejected": -11.501545906066895,
+      "semantic_entropy": 0.0009456650586798787,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 28.922275115926304,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": 1.585463047027588,
+      "logits/rejected": 1.6918182373046875,
+      "logps/chosen": -10.196683883666992,
+      "logps/rejected": -11.841047286987305,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.196683883666992,
+      "rewards/margins": 1.6443649530410767,
+      "rewards/rejected": -11.841047286987305,
+      "semantic_entropy": 0.0009173370199277997,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 28.08257045971241,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": 1.548707365989685,
+      "logits/rejected": 1.6452610492706299,
+      "logps/chosen": -10.007037162780762,
+      "logps/rejected": -11.586597442626953,
+      "loss": 0.3652,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.007037162780762,
+      "rewards/margins": 1.5795583724975586,
+      "rewards/rejected": -11.586597442626953,
+      "semantic_entropy": 0.0010930246207863092,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 27.517337996671454,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": 1.50733482837677,
+      "logits/rejected": 1.589125394821167,
+      "logps/chosen": -10.230108261108398,
+      "logps/rejected": -11.490400314331055,
+      "loss": 0.4204,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -10.230108261108398,
+      "rewards/margins": 1.2602918148040771,
+      "rewards/rejected": -11.490400314331055,
+      "semantic_entropy": 0.0008926793816499412,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 18.53376925534759,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": 1.50508713722229,
+      "logits/rejected": 1.5913264751434326,
+      "logps/chosen": -10.254201889038086,
+      "logps/rejected": -11.766775131225586,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.254201889038086,
+      "rewards/margins": 1.512573480606079,
+      "rewards/rejected": -11.766775131225586,
+      "semantic_entropy": 0.0008712027338333428,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 29.736383696491377,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": 1.5540765523910522,
+      "logits/rejected": 1.6055597066879272,
+      "logps/chosen": -10.118463516235352,
+      "logps/rejected": -11.558184623718262,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.118463516235352,
+      "rewards/margins": 1.4397218227386475,
+      "rewards/rejected": -11.558184623718262,
+      "semantic_entropy": 0.0009123588679358363,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 27.051760502901796,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": 1.5811470746994019,
+      "logits/rejected": 1.64520263671875,
+      "logps/chosen": -10.145907402038574,
+      "logps/rejected": -11.47169303894043,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.145907402038574,
+      "rewards/margins": 1.325786828994751,
+      "rewards/rejected": -11.47169303894043,
+      "semantic_entropy": 0.0010149618610739708,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 21.285196697912564,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": 1.5973271131515503,
+      "logits/rejected": 1.6571729183197021,
+      "logps/chosen": -10.162995338439941,
+      "logps/rejected": -11.612829208374023,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.162995338439941,
+      "rewards/margins": 1.4498339891433716,
+      "rewards/rejected": -11.612829208374023,
+      "semantic_entropy": 0.0010343410540372133,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 26.329625603153108,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": 1.5865263938903809,
+      "logits/rejected": 1.626970887184143,
+      "logps/chosen": -9.906156539916992,
+      "logps/rejected": -11.410951614379883,
+      "loss": 0.387,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.906156539916992,
+      "rewards/margins": 1.5047956705093384,
+      "rewards/rejected": -11.410951614379883,
+      "semantic_entropy": 0.0013154713669791818,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 17.59802845768495,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": 1.5388119220733643,
+      "logits/rejected": 1.6437499523162842,
+      "logps/chosen": -10.019671440124512,
+      "logps/rejected": -11.555891036987305,
+      "loss": 0.3831,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.019671440124512,
+      "rewards/margins": 1.536219835281372,
+      "rewards/rejected": -11.555891036987305,
+      "semantic_entropy": 0.0010032164864242077,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 27.433272508628217,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": 1.5722072124481201,
+      "logits/rejected": 1.6376203298568726,
+      "logps/chosen": -10.016858100891113,
+      "logps/rejected": -11.583300590515137,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.016858100891113,
+      "rewards/margins": 1.566442847251892,
+      "rewards/rejected": -11.583300590515137,
+      "semantic_entropy": 0.001173605676740408,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 24.56912782860724,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": 1.5965417623519897,
+      "logits/rejected": 1.6971406936645508,
+      "logps/chosen": -10.133933067321777,
+      "logps/rejected": -11.667776107788086,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.133933067321777,
+      "rewards/margins": 1.533843994140625,
+      "rewards/rejected": -11.667776107788086,
+      "semantic_entropy": 0.0010934568708762527,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 24.437402776876635,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": 1.5532182455062866,
+      "logits/rejected": 1.6137325763702393,
+      "logps/chosen": -10.124250411987305,
+      "logps/rejected": -11.495368957519531,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.124250411987305,
+      "rewards/margins": 1.3711193799972534,
+      "rewards/rejected": -11.495368957519531,
+      "semantic_entropy": 0.0010359478183090687,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 24.396430519150893,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": 1.5740119218826294,
+      "logits/rejected": 1.6382734775543213,
+      "logps/chosen": -9.948430061340332,
+      "logps/rejected": -11.49321460723877,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.948430061340332,
+      "rewards/margins": 1.5447834730148315,
+      "rewards/rejected": -11.49321460723877,
+      "semantic_entropy": 0.0012237438932061195,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 32.194888859048426,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": 1.5697987079620361,
+      "logits/rejected": 1.6561062335968018,
+      "logps/chosen": -10.188702583312988,
+      "logps/rejected": -11.645292282104492,
+      "loss": 0.4143,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.188702583312988,
+      "rewards/margins": 1.4565904140472412,
+      "rewards/rejected": -11.645292282104492,
+      "semantic_entropy": 0.0011257819132879376,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 20.235465282279524,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": 1.5929367542266846,
+      "logits/rejected": 1.624206304550171,
+      "logps/chosen": -10.148530960083008,
+      "logps/rejected": -11.857019424438477,
+      "loss": 0.3131,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -10.148530960083008,
+      "rewards/margins": 1.7084884643554688,
+      "rewards/rejected": -11.857019424438477,
+      "semantic_entropy": 0.0009746078285388649,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 28.920023305548266,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": 1.5221036672592163,
+      "logits/rejected": 1.6088695526123047,
+      "logps/chosen": -10.077486038208008,
+      "logps/rejected": -11.675275802612305,
+      "loss": 0.3541,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.077486038208008,
+      "rewards/margins": 1.5977914333343506,
+      "rewards/rejected": -11.675275802612305,
+      "semantic_entropy": 0.0009062701719813049,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 26.002031841446875,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": 1.5474283695220947,
+      "logits/rejected": 1.6092983484268188,
+      "logps/chosen": -10.160518646240234,
+      "logps/rejected": -11.628789901733398,
+      "loss": 0.4573,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.160518646240234,
+      "rewards/margins": 1.468271017074585,
+      "rewards/rejected": -11.628789901733398,
+      "semantic_entropy": 0.0012058538850396872,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 24.858127180108646,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": 1.5863932371139526,
+      "logits/rejected": 1.6843297481536865,
+      "logps/chosen": -10.116307258605957,
+      "logps/rejected": -11.749727249145508,
+      "loss": 0.3568,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.116307258605957,
+      "rewards/margins": 1.6334211826324463,
+      "rewards/rejected": -11.749727249145508,
+      "semantic_entropy": 0.0011141184950247407,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 28.184650058916827,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": 1.5939695835113525,
+      "logits/rejected": 1.6244093179702759,
+      "logps/chosen": -10.198641777038574,
+      "logps/rejected": -11.6402587890625,
+      "loss": 0.3699,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.198641777038574,
+      "rewards/margins": 1.4416176080703735,
+      "rewards/rejected": -11.6402587890625,
+      "semantic_entropy": 0.0011869249865412712,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 20.929202572709965,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": 1.5695644617080688,
+      "logits/rejected": 1.616767168045044,
+      "logps/chosen": -10.1691255569458,
+      "logps/rejected": -11.662959098815918,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.1691255569458,
+      "rewards/margins": 1.4938335418701172,
+      "rewards/rejected": -11.662959098815918,
+      "semantic_entropy": 0.0010471201967447996,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 29.244095253670732,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": 1.5500667095184326,
+      "logits/rejected": 1.5883280038833618,
+      "logps/chosen": -10.329306602478027,
+      "logps/rejected": -11.774639129638672,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.329306602478027,
+      "rewards/margins": 1.4453322887420654,
+      "rewards/rejected": -11.774639129638672,
+      "semantic_entropy": 0.000884855748154223,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 26.28630897005537,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": 1.5725198984146118,
+      "logits/rejected": 1.661706566810608,
+      "logps/chosen": -10.130839347839355,
+      "logps/rejected": -11.913408279418945,
+      "loss": 0.3388,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.130839347839355,
+      "rewards/margins": 1.7825679779052734,
+      "rewards/rejected": -11.913408279418945,
+      "semantic_entropy": 0.0010064066154882312,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 22.209186339803793,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": 1.6010830402374268,
+      "logits/rejected": 1.6500654220581055,
+      "logps/chosen": -10.342905044555664,
+      "logps/rejected": -11.871801376342773,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.342905044555664,
+      "rewards/margins": 1.5288969278335571,
+      "rewards/rejected": -11.871801376342773,
+      "semantic_entropy": 0.0010242112912237644,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 21.220515390674517,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": 1.5785582065582275,
+      "logits/rejected": 1.6533657312393188,
+      "logps/chosen": -9.987582206726074,
+      "logps/rejected": -11.735966682434082,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -9.987582206726074,
+      "rewards/margins": 1.7483835220336914,
+      "rewards/rejected": -11.735966682434082,
+      "semantic_entropy": 0.0010586322750896215,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 25.089529408030568,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": 1.5576601028442383,
+      "logits/rejected": 1.637877106666565,
+      "logps/chosen": -10.180359840393066,
+      "logps/rejected": -11.435386657714844,
+      "loss": 0.4298,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.180359840393066,
+      "rewards/margins": 1.2550264596939087,
+      "rewards/rejected": -11.435386657714844,
+      "semantic_entropy": 0.00091672467533499,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 27.697436892371897,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": 1.5720164775848389,
+      "logits/rejected": 1.6422309875488281,
+      "logps/chosen": -10.055976867675781,
+      "logps/rejected": -11.807050704956055,
+      "loss": 0.3654,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.055976867675781,
+      "rewards/margins": 1.7510731220245361,
+      "rewards/rejected": -11.807050704956055,
+      "semantic_entropy": 0.00120059703476727,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 27.98922690744636,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": 1.5998647212982178,
+      "logits/rejected": 1.659576177597046,
+      "logps/chosen": -10.253899574279785,
+      "logps/rejected": -11.79109001159668,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.253899574279785,
+      "rewards/margins": 1.5371911525726318,
+      "rewards/rejected": -11.79109001159668,
+      "semantic_entropy": 0.0009287342545576394,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 22.837576546184515,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": 1.5321710109710693,
+      "logits/rejected": 1.6386340856552124,
+      "logps/chosen": -10.002676010131836,
+      "logps/rejected": -11.531843185424805,
+      "loss": 0.3674,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.002676010131836,
+      "rewards/margins": 1.5291664600372314,
+      "rewards/rejected": -11.531843185424805,
+      "semantic_entropy": 0.0011454729828983545,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 31.061808262171656,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": 1.5541818141937256,
+      "logits/rejected": 1.6007535457611084,
+      "logps/chosen": -10.152975082397461,
+      "logps/rejected": -11.46742057800293,
+      "loss": 0.431,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.152975082397461,
+      "rewards/margins": 1.3144452571868896,
+      "rewards/rejected": -11.46742057800293,
+      "semantic_entropy": 0.0009469335782341659,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 32.41103980274439,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": 1.5465234518051147,
+      "logits/rejected": 1.599394679069519,
+      "logps/chosen": -10.142258644104004,
+      "logps/rejected": -11.76301383972168,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.142258644104004,
+      "rewards/margins": 1.6207565069198608,
+      "rewards/rejected": -11.76301383972168,
+      "semantic_entropy": 0.0014503062702715397,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 23.84751877309452,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": 1.596401333808899,
+      "logits/rejected": 1.660299301147461,
+      "logps/chosen": -10.249316215515137,
+      "logps/rejected": -11.84397029876709,
+      "loss": 0.3672,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.249316215515137,
+      "rewards/margins": 1.5946543216705322,
+      "rewards/rejected": -11.84397029876709,
+      "semantic_entropy": 0.0010910606943070889,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 17.106149733952037,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": 1.590715765953064,
+      "logits/rejected": 1.6165393590927124,
+      "logps/chosen": -9.983671188354492,
+      "logps/rejected": -11.444806098937988,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.983671188354492,
+      "rewards/margins": 1.4611353874206543,
+      "rewards/rejected": -11.444806098937988,
+      "semantic_entropy": 0.001151026925072074,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": 1.6280796527862549,
+      "eval_logits/rejected": 1.6694473028182983,
+      "eval_logps/chosen": -10.298751831054688,
+      "eval_logps/rejected": -11.491374969482422,
+      "eval_loss": 0.5416285991668701,
+      "eval_rewards/accuracies": 0.7240356206893921,
+      "eval_rewards/chosen": -10.298751831054688,
+      "eval_rewards/margins": 1.1926239728927612,
+      "eval_rewards/rejected": -11.491374969482422,
+      "eval_runtime": 34.8117,
+      "eval_samples_per_second": 38.636,
+      "eval_semantic_entropy": 0.001040897099301219,
+      "eval_steps_per_second": 9.681,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 31.497403656545167,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": 1.576177716255188,
+      "logits/rejected": 1.6267242431640625,
+      "logps/chosen": -10.096414566040039,
+      "logps/rejected": -11.5610933303833,
+      "loss": 0.4333,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.096414566040039,
+      "rewards/margins": 1.4646787643432617,
+      "rewards/rejected": -11.5610933303833,
+      "semantic_entropy": 0.0010507599217817187,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 28.70320202082784,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": 1.5821866989135742,
+      "logits/rejected": 1.6370443105697632,
+      "logps/chosen": -10.019612312316895,
+      "logps/rejected": -11.597597122192383,
+      "loss": 0.3876,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.019612312316895,
+      "rewards/margins": 1.5779845714569092,
+      "rewards/rejected": -11.597597122192383,
+      "semantic_entropy": 0.0011676338035613298,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 21.39178758205242,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": 1.6148006916046143,
+      "logits/rejected": 1.652077317237854,
+      "logps/chosen": -9.876248359680176,
+      "logps/rejected": -11.604894638061523,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.876248359680176,
+      "rewards/margins": 1.7286474704742432,
+      "rewards/rejected": -11.604894638061523,
+      "semantic_entropy": 0.001374484389089048,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 29.384896205636583,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": 1.5841318368911743,
+      "logits/rejected": 1.6482540369033813,
+      "logps/chosen": -10.043427467346191,
+      "logps/rejected": -11.435232162475586,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.043427467346191,
+      "rewards/margins": 1.3918060064315796,
+      "rewards/rejected": -11.435232162475586,
+      "semantic_entropy": 0.001155240461230278,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 36.76118710657995,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": 1.6225206851959229,
+      "logits/rejected": 1.7067546844482422,
+      "logps/chosen": -10.0739164352417,
+      "logps/rejected": -11.532676696777344,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.0739164352417,
+      "rewards/margins": 1.4587604999542236,
+      "rewards/rejected": -11.532676696777344,
+      "semantic_entropy": 0.001224294537678361,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 30.59277088842713,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": 1.5949815511703491,
+      "logits/rejected": 1.6801363229751587,
+      "logps/chosen": -10.12843132019043,
+      "logps/rejected": -11.617376327514648,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.12843132019043,
+      "rewards/margins": 1.4889460802078247,
+      "rewards/rejected": -11.617376327514648,
+      "semantic_entropy": 0.001076376996934414,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 24.974056349149848,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": 1.518202781677246,
+      "logits/rejected": 1.611853837966919,
+      "logps/chosen": -9.976676940917969,
+      "logps/rejected": -11.617460250854492,
+      "loss": 0.328,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -9.976676940917969,
+      "rewards/margins": 1.6407829523086548,
+      "rewards/rejected": -11.617460250854492,
+      "semantic_entropy": 0.0011249089147895575,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 30.992665327296987,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": 1.5575978755950928,
+      "logits/rejected": 1.6567716598510742,
+      "logps/chosen": -10.085691452026367,
+      "logps/rejected": -11.682951927185059,
+      "loss": 0.3575,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.085691452026367,
+      "rewards/margins": 1.597259759902954,
+      "rewards/rejected": -11.682951927185059,
+      "semantic_entropy": 0.001052472973242402,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 31.941453658254563,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": 1.5707769393920898,
+      "logits/rejected": 1.6170399188995361,
+      "logps/chosen": -9.970999717712402,
+      "logps/rejected": -11.392215728759766,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -9.970999717712402,
+      "rewards/margins": 1.421215295791626,
+      "rewards/rejected": -11.392215728759766,
+      "semantic_entropy": 0.00112369772978127,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 31.203432233864095,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": 1.5358566045761108,
+      "logits/rejected": 1.6192134618759155,
+      "logps/chosen": -10.108580589294434,
+      "logps/rejected": -11.670511245727539,
+      "loss": 0.396,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.108580589294434,
+      "rewards/margins": 1.5619298219680786,
+      "rewards/rejected": -11.670511245727539,
+      "semantic_entropy": 0.0010274298256263137,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 30.387922911554366,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": 1.619341492652893,
+      "logits/rejected": 1.682959794998169,
+      "logps/chosen": -10.036428451538086,
+      "logps/rejected": -11.582202911376953,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.036428451538086,
+      "rewards/margins": 1.5457748174667358,
+      "rewards/rejected": -11.582202911376953,
+      "semantic_entropy": 0.0011530378833413124,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 32.65734770164333,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": 1.5509191751480103,
+      "logits/rejected": 1.5898683071136475,
+      "logps/chosen": -9.97402286529541,
+      "logps/rejected": -11.500492095947266,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.97402286529541,
+      "rewards/margins": 1.5264681577682495,
+      "rewards/rejected": -11.500492095947266,
+      "semantic_entropy": 0.001599832670763135,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 17.73320803212675,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": 1.6105467081069946,
+      "logits/rejected": 1.6437714099884033,
+      "logps/chosen": -10.296040534973145,
+      "logps/rejected": -11.82190990447998,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.296040534973145,
+      "rewards/margins": 1.5258680582046509,
+      "rewards/rejected": -11.82190990447998,
+      "semantic_entropy": 0.0009045203914865851,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 19.353812166691853,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": 1.6078459024429321,
+      "logits/rejected": 1.6979691982269287,
+      "logps/chosen": -9.930005073547363,
+      "logps/rejected": -11.7776460647583,
+      "loss": 0.2987,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -9.930005073547363,
+      "rewards/margins": 1.8476402759552002,
+      "rewards/rejected": -11.7776460647583,
+      "semantic_entropy": 0.001331482781097293,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 28.56331094700217,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": 1.5709428787231445,
+      "logits/rejected": 1.6303266286849976,
+      "logps/chosen": -9.989324569702148,
+      "logps/rejected": -11.608041763305664,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -9.989324569702148,
+      "rewards/margins": 1.6187162399291992,
+      "rewards/rejected": -11.608041763305664,
+      "semantic_entropy": 0.0010678318794816732,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 20.17223944967117,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": 1.6492599248886108,
+      "logits/rejected": 1.6782468557357788,
+      "logps/chosen": -10.164798736572266,
+      "logps/rejected": -11.784120559692383,
+      "loss": 0.3672,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.164798736572266,
+      "rewards/margins": 1.6193220615386963,
+      "rewards/rejected": -11.784120559692383,
+      "semantic_entropy": 0.0010477247415110469,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 23.283717549959018,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": 1.5857760906219482,
+      "logits/rejected": 1.649200201034546,
+      "logps/chosen": -10.31878662109375,
+      "logps/rejected": -11.922708511352539,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.31878662109375,
+      "rewards/margins": 1.603921890258789,
+      "rewards/rejected": -11.922708511352539,
+      "semantic_entropy": 0.000815052364487201,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 26.25632574029958,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": 1.5767600536346436,
+      "logits/rejected": 1.6210075616836548,
+      "logps/chosen": -10.138204574584961,
+      "logps/rejected": -11.657526016235352,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.138204574584961,
+      "rewards/margins": 1.519321084022522,
+      "rewards/rejected": -11.657526016235352,
+      "semantic_entropy": 0.0009958173613995314,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 21.26093578562833,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": 1.5494698286056519,
+      "logits/rejected": 1.627808928489685,
+      "logps/chosen": -10.259526252746582,
+      "logps/rejected": -11.573699951171875,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.259526252746582,
+      "rewards/margins": 1.3141748905181885,
+      "rewards/rejected": -11.573699951171875,
+      "semantic_entropy": 0.0014686653157696128,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 22.57917348493258,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": 1.5637296438217163,
+      "logits/rejected": 1.6783336400985718,
+      "logps/chosen": -10.307500839233398,
+      "logps/rejected": -11.929152488708496,
+      "loss": 0.3479,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.307500839233398,
+      "rewards/margins": 1.6216518878936768,
+      "rewards/rejected": -11.929152488708496,
+      "semantic_entropy": 0.0008739338372834027,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 24.630109593218762,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": 1.5679190158843994,
+      "logits/rejected": 1.661757230758667,
+      "logps/chosen": -10.260305404663086,
+      "logps/rejected": -11.89435863494873,
+      "loss": 0.3299,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.260305404663086,
+      "rewards/margins": 1.6340538263320923,
+      "rewards/rejected": -11.89435863494873,
+      "semantic_entropy": 0.0009901316370815039,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 25.90014461239659,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": 1.5774565935134888,
+      "logits/rejected": 1.6124422550201416,
+      "logps/chosen": -10.210862159729004,
+      "logps/rejected": -11.889470100402832,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.210862159729004,
+      "rewards/margins": 1.6786079406738281,
+      "rewards/rejected": -11.889470100402832,
+      "semantic_entropy": 0.0009773386409506202,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 28.60310676993165,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": 1.5957114696502686,
+      "logits/rejected": 1.676044225692749,
+      "logps/chosen": -10.24717903137207,
+      "logps/rejected": -11.700576782226562,
+      "loss": 0.4215,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.24717903137207,
+      "rewards/margins": 1.4533963203430176,
+      "rewards/rejected": -11.700576782226562,
+      "semantic_entropy": 0.0011988317128270864,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 24.442604383823113,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": 1.5716701745986938,
+      "logits/rejected": 1.639828085899353,
+      "logps/chosen": -10.324374198913574,
+      "logps/rejected": -11.902523040771484,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.324374198913574,
+      "rewards/margins": 1.5781484842300415,
+      "rewards/rejected": -11.902523040771484,
+      "semantic_entropy": 0.0008550761267542839,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 22.24898411949683,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": 1.510594129562378,
+      "logits/rejected": 1.63089919090271,
+      "logps/chosen": -10.221429824829102,
+      "logps/rejected": -11.841843605041504,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.221429824829102,
+      "rewards/margins": 1.620413064956665,
+      "rewards/rejected": -11.841843605041504,
+      "semantic_entropy": 0.0009447725606150925,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 21.845049626995486,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": 1.5962519645690918,
+      "logits/rejected": 1.6844122409820557,
+      "logps/chosen": -10.277950286865234,
+      "logps/rejected": -11.939562797546387,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.277950286865234,
+      "rewards/margins": 1.6616131067276,
+      "rewards/rejected": -11.939562797546387,
+      "semantic_entropy": 0.0010203439742326736,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 38.640628706397905,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": 1.573960542678833,
+      "logits/rejected": 1.6671758890151978,
+      "logps/chosen": -10.220738410949707,
+      "logps/rejected": -11.771955490112305,
+      "loss": 0.4215,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -10.220738410949707,
+      "rewards/margins": 1.5512182712554932,
+      "rewards/rejected": -11.771955490112305,
+      "semantic_entropy": 0.0009818064281716943,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 21.91879334840554,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": 1.5751703977584839,
+      "logits/rejected": 1.6245527267456055,
+      "logps/chosen": -10.107606887817383,
+      "logps/rejected": -11.549296379089355,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.107606887817383,
+      "rewards/margins": 1.441690444946289,
+      "rewards/rejected": -11.549296379089355,
+      "semantic_entropy": 0.0014187381602823734,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 25.229586751310848,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": 1.5829904079437256,
+      "logits/rejected": 1.6850636005401611,
+      "logps/chosen": -10.25570297241211,
+      "logps/rejected": -11.90019416809082,
+      "loss": 0.3545,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.25570297241211,
+      "rewards/margins": 1.644491195678711,
+      "rewards/rejected": -11.90019416809082,
+      "semantic_entropy": 0.000863713794387877,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 28.445600756091924,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": 1.6099693775177002,
+      "logits/rejected": 1.68081796169281,
+      "logps/chosen": -10.22422981262207,
+      "logps/rejected": -11.845881462097168,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.22422981262207,
+      "rewards/margins": 1.6216522455215454,
+      "rewards/rejected": -11.845881462097168,
+      "semantic_entropy": 0.0010618824744597077,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 29.406307980418067,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": 1.576688289642334,
+      "logits/rejected": 1.6222105026245117,
+      "logps/chosen": -10.334737777709961,
+      "logps/rejected": -11.768031120300293,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.334737777709961,
+      "rewards/margins": 1.433293104171753,
+      "rewards/rejected": -11.768031120300293,
+      "semantic_entropy": 0.0008148913038894534,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 26.601157297466546,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": 1.589557409286499,
+      "logits/rejected": 1.6633790731430054,
+      "logps/chosen": -10.160542488098145,
+      "logps/rejected": -11.787088394165039,
+      "loss": 0.387,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.160542488098145,
+      "rewards/margins": 1.6265466213226318,
+      "rewards/rejected": -11.787088394165039,
+      "semantic_entropy": 0.0013252630596980453,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 34.58651300526,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": 1.5564419031143188,
+      "logits/rejected": 1.6088470220565796,
+      "logps/chosen": -10.14398193359375,
+      "logps/rejected": -11.792848587036133,
+      "loss": 0.3465,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.14398193359375,
+      "rewards/margins": 1.6488673686981201,
+      "rewards/rejected": -11.792848587036133,
+      "semantic_entropy": 0.000979890814051032,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 25.40781027004546,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": 1.6011059284210205,
+      "logits/rejected": 1.660117506980896,
+      "logps/chosen": -9.878233909606934,
+      "logps/rejected": -11.536373138427734,
+      "loss": 0.3453,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -9.878233909606934,
+      "rewards/margins": 1.658138632774353,
+      "rewards/rejected": -11.536373138427734,
+      "semantic_entropy": 0.0011360698845237494,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 24.820874129201034,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": 1.589334487915039,
+      "logits/rejected": 1.6348495483398438,
+      "logps/chosen": -10.285005569458008,
+      "logps/rejected": -11.802273750305176,
+      "loss": 0.3547,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.285005569458008,
+      "rewards/margins": 1.5172683000564575,
+      "rewards/rejected": -11.802273750305176,
+      "semantic_entropy": 0.0007844140054658055,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 20.548755094555784,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": 1.6208778619766235,
+      "logits/rejected": 1.6700575351715088,
+      "logps/chosen": -10.401395797729492,
+      "logps/rejected": -11.833595275878906,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.401395797729492,
+      "rewards/margins": 1.432199239730835,
+      "rewards/rejected": -11.833595275878906,
+      "semantic_entropy": 0.0007872053538449109,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 27.736484595147676,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": 1.5939035415649414,
+      "logits/rejected": 1.6784961223602295,
+      "logps/chosen": -9.981462478637695,
+      "logps/rejected": -11.691766738891602,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.981462478637695,
+      "rewards/margins": 1.7103036642074585,
+      "rewards/rejected": -11.691766738891602,
+      "semantic_entropy": 0.0013304570456966758,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 39.161026020871674,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": 1.5560963153839111,
+      "logits/rejected": 1.628401756286621,
+      "logps/chosen": -10.073539733886719,
+      "logps/rejected": -11.771159172058105,
+      "loss": 0.3395,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.073539733886719,
+      "rewards/margins": 1.6976196765899658,
+      "rewards/rejected": -11.771159172058105,
+      "semantic_entropy": 0.001218894263729453,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 28.117024768885692,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": 1.552726149559021,
+      "logits/rejected": 1.5963780879974365,
+      "logps/chosen": -10.304581642150879,
+      "logps/rejected": -11.607521057128906,
+      "loss": 0.453,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.304581642150879,
+      "rewards/margins": 1.3029396533966064,
+      "rewards/rejected": -11.607521057128906,
+      "semantic_entropy": 0.0008688464877195656,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 22.283406629975044,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": 1.5720824003219604,
+      "logits/rejected": 1.6107498407363892,
+      "logps/chosen": -10.217363357543945,
+      "logps/rejected": -11.781001091003418,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.217363357543945,
+      "rewards/margins": 1.563639760017395,
+      "rewards/rejected": -11.781001091003418,
+      "semantic_entropy": 0.001032182713970542,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 29.184149011680613,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": 1.5839941501617432,
+      "logits/rejected": 1.6152938604354858,
+      "logps/chosen": -10.263362884521484,
+      "logps/rejected": -11.636185646057129,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.263362884521484,
+      "rewards/margins": 1.3728222846984863,
+      "rewards/rejected": -11.636185646057129,
+      "semantic_entropy": 0.0009062589961104095,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 26.764838572629287,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": 1.5441625118255615,
+      "logits/rejected": 1.6142288446426392,
+      "logps/chosen": -10.215639114379883,
+      "logps/rejected": -11.769659996032715,
+      "loss": 0.3546,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.215639114379883,
+      "rewards/margins": 1.5540201663970947,
+      "rewards/rejected": -11.769659996032715,
+      "semantic_entropy": 0.0010565847624093294,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 27.302837410651627,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": 1.5716558694839478,
+      "logits/rejected": 1.6289341449737549,
+      "logps/chosen": -10.11702823638916,
+      "logps/rejected": -11.728710174560547,
+      "loss": 0.3693,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.11702823638916,
+      "rewards/margins": 1.611682653427124,
+      "rewards/rejected": -11.728710174560547,
+      "semantic_entropy": 0.0009398089023306966,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 38.21211029393227,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": 1.5621964931488037,
+      "logits/rejected": 1.6454849243164062,
+      "logps/chosen": -10.148094177246094,
+      "logps/rejected": -11.587119102478027,
+      "loss": 0.4204,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.148094177246094,
+      "rewards/margins": 1.4390251636505127,
+      "rewards/rejected": -11.587119102478027,
+      "semantic_entropy": 0.0009816800011321902,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 20.873006090490513,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": 1.585795521736145,
+      "logits/rejected": 1.65435791015625,
+      "logps/chosen": -10.318081855773926,
+      "logps/rejected": -11.786093711853027,
+      "loss": 0.4085,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.318081855773926,
+      "rewards/margins": 1.4680125713348389,
+      "rewards/rejected": -11.786093711853027,
+      "semantic_entropy": 0.0007593315676786005,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 26.062748531819903,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": 1.6093727350234985,
+      "logits/rejected": 1.6642532348632812,
+      "logps/chosen": -10.124918937683105,
+      "logps/rejected": -11.888010025024414,
+      "loss": 0.3346,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.124918937683105,
+      "rewards/margins": 1.7630901336669922,
+      "rewards/rejected": -11.888010025024414,
+      "semantic_entropy": 0.00110786990262568,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 32.74355229193564,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": 1.5574390888214111,
+      "logits/rejected": 1.645414113998413,
+      "logps/chosen": -10.405125617980957,
+      "logps/rejected": -11.87415885925293,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.405125617980957,
+      "rewards/margins": 1.4690325260162354,
+      "rewards/rejected": -11.87415885925293,
+      "semantic_entropy": 0.0009024108876474202,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 26.19993502715388,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": 1.6740386486053467,
+      "logits/rejected": 1.6717405319213867,
+      "logps/chosen": -10.36558723449707,
+      "logps/rejected": -11.81406307220459,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.36558723449707,
+      "rewards/margins": 1.4484764337539673,
+      "rewards/rejected": -11.81406307220459,
+      "semantic_entropy": 0.0008315413142554462,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 27.279500634177804,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": 1.6468241214752197,
+      "logits/rejected": 1.6623125076293945,
+      "logps/chosen": -10.261005401611328,
+      "logps/rejected": -11.790407180786133,
+      "loss": 0.3629,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.261005401611328,
+      "rewards/margins": 1.5294020175933838,
+      "rewards/rejected": -11.790407180786133,
+      "semantic_entropy": 0.0009241552907042205,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 24.222144852253166,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": 1.5816023349761963,
+      "logits/rejected": 1.6238762140274048,
+      "logps/chosen": -10.056999206542969,
+      "logps/rejected": -11.567682266235352,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.056999206542969,
+      "rewards/margins": 1.5106838941574097,
+      "rewards/rejected": -11.567682266235352,
+      "semantic_entropy": 0.0011438427027314901,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 26.367225241411354,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": 1.5425548553466797,
+      "logits/rejected": 1.6112905740737915,
+      "logps/chosen": -9.978961944580078,
+      "logps/rejected": -11.640503883361816,
+      "loss": 0.3578,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.978961944580078,
+      "rewards/margins": 1.6615406274795532,
+      "rewards/rejected": -11.640503883361816,
+      "semantic_entropy": 0.0010940327774733305,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 28.096074059780435,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": 1.5880234241485596,
+      "logits/rejected": 1.6233491897583008,
+      "logps/chosen": -10.134669303894043,
+      "logps/rejected": -11.51002311706543,
+      "loss": 0.4548,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.134669303894043,
+      "rewards/margins": 1.375354528427124,
+      "rewards/rejected": -11.51002311706543,
+      "semantic_entropy": 0.0010770043591037393,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 30.20466558005367,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": 1.6001932621002197,
+      "logits/rejected": 1.6647536754608154,
+      "logps/chosen": -9.882922172546387,
+      "logps/rejected": -11.453819274902344,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.882922172546387,
+      "rewards/margins": 1.5708967447280884,
+      "rewards/rejected": -11.453819274902344,
+      "semantic_entropy": 0.001264480291865766,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 31.447363091532164,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": 1.6478302478790283,
+      "logits/rejected": 1.7154209613800049,
+      "logps/chosen": -10.129390716552734,
+      "logps/rejected": -11.509721755981445,
+      "loss": 0.401,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.129390716552734,
+      "rewards/margins": 1.3803313970565796,
+      "rewards/rejected": -11.509721755981445,
+      "semantic_entropy": 0.0010178691009059548,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 35.80875454608141,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": 1.645939588546753,
+      "logits/rejected": 1.6861419677734375,
+      "logps/chosen": -10.13318157196045,
+      "logps/rejected": -11.375663757324219,
+      "loss": 0.4645,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.13318157196045,
+      "rewards/margins": 1.2424815893173218,
+      "rewards/rejected": -11.375663757324219,
+      "semantic_entropy": 0.001157456892542541,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 29.2295015988339,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": 1.600102186203003,
+      "logits/rejected": 1.665477991104126,
+      "logps/chosen": -10.163004875183105,
+      "logps/rejected": -11.72107982635498,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.163004875183105,
+      "rewards/margins": 1.558074712753296,
+      "rewards/rejected": -11.72107982635498,
+      "semantic_entropy": 0.000961720768827945,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 34.89366225760279,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": 1.6140092611312866,
+      "logits/rejected": 1.6532926559448242,
+      "logps/chosen": -10.069607734680176,
+      "logps/rejected": -11.870946884155273,
+      "loss": 0.3151,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -10.069607734680176,
+      "rewards/margins": 1.8013403415679932,
+      "rewards/rejected": -11.870946884155273,
+      "semantic_entropy": 0.0011587004410102963,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 28.79715741996094,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": 1.5889379978179932,
+      "logits/rejected": 1.6730520725250244,
+      "logps/chosen": -10.315347671508789,
+      "logps/rejected": -11.844008445739746,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.315347671508789,
+      "rewards/margins": 1.5286613702774048,
+      "rewards/rejected": -11.844008445739746,
+      "semantic_entropy": 0.0009549719397909939,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 27.167907920059488,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": 1.5163604021072388,
+      "logits/rejected": 1.613382339477539,
+      "logps/chosen": -10.050630569458008,
+      "logps/rejected": -11.761641502380371,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.050630569458008,
+      "rewards/margins": 1.7110118865966797,
+      "rewards/rejected": -11.761641502380371,
+      "semantic_entropy": 0.0013580418890342116,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 24.03630557922596,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": 1.608271837234497,
+      "logits/rejected": 1.6711280345916748,
+      "logps/chosen": -10.284029960632324,
+      "logps/rejected": -11.932718276977539,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.284029960632324,
+      "rewards/margins": 1.648687720298767,
+      "rewards/rejected": -11.932718276977539,
+      "semantic_entropy": 0.0011336029274389148,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 33.36268258503627,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": 1.571376085281372,
+      "logits/rejected": 1.633230209350586,
+      "logps/chosen": -10.418757438659668,
+      "logps/rejected": -11.856203079223633,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.418757438659668,
+      "rewards/margins": 1.4374463558197021,
+      "rewards/rejected": -11.856203079223633,
+      "semantic_entropy": 0.0009451628429815173,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 26.24567378800384,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": 1.5546691417694092,
+      "logits/rejected": 1.6034162044525146,
+      "logps/chosen": -10.047476768493652,
+      "logps/rejected": -11.581827163696289,
+      "loss": 0.4135,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.047476768493652,
+      "rewards/margins": 1.5343519449234009,
+      "rewards/rejected": -11.581827163696289,
+      "semantic_entropy": 0.0010649125324562192,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 31.11139831191975,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": 1.5543608665466309,
+      "logits/rejected": 1.6508735418319702,
+      "logps/chosen": -10.119691848754883,
+      "logps/rejected": -11.670234680175781,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.119691848754883,
+      "rewards/margins": 1.5505428314208984,
+      "rewards/rejected": -11.670234680175781,
+      "semantic_entropy": 0.0011525005102157593,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 34.069069961672945,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": 1.5943682193756104,
+      "logits/rejected": 1.6631578207015991,
+      "logps/chosen": -10.102033615112305,
+      "logps/rejected": -11.633230209350586,
+      "loss": 0.3897,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.102033615112305,
+      "rewards/margins": 1.5311968326568604,
+      "rewards/rejected": -11.633230209350586,
+      "semantic_entropy": 0.0010671000927686691,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 40.22036012157819,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": 1.5197334289550781,
+      "logits/rejected": 1.648328423500061,
+      "logps/chosen": -10.165567398071289,
+      "logps/rejected": -11.725723266601562,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.165567398071289,
+      "rewards/margins": 1.560157060623169,
+      "rewards/rejected": -11.725723266601562,
+      "semantic_entropy": 0.0009594460716471076,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 27.082935740015643,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": 1.542008638381958,
+      "logits/rejected": 1.636115312576294,
+      "logps/chosen": -10.095491409301758,
+      "logps/rejected": -11.64437198638916,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.095491409301758,
+      "rewards/margins": 1.548882246017456,
+      "rewards/rejected": -11.64437198638916,
+      "semantic_entropy": 0.0011894134804606438,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 37.75956895470681,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": 1.6362298727035522,
+      "logits/rejected": 1.6833397150039673,
+      "logps/chosen": -10.217446327209473,
+      "logps/rejected": -11.570792198181152,
+      "loss": 0.4176,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.217446327209473,
+      "rewards/margins": 1.353346586227417,
+      "rewards/rejected": -11.570792198181152,
+      "semantic_entropy": 0.0012145208893343806,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 22.51758773135186,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": 1.6473830938339233,
+      "logits/rejected": 1.7397950887680054,
+      "logps/chosen": -10.32457447052002,
+      "logps/rejected": -11.925352096557617,
+      "loss": 0.357,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.32457447052002,
+      "rewards/margins": 1.6007791757583618,
+      "rewards/rejected": -11.925352096557617,
+      "semantic_entropy": 0.0008410528535023332,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 41.649501318426786,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": 1.6099411249160767,
+      "logits/rejected": 1.6837127208709717,
+      "logps/chosen": -10.264101028442383,
+      "logps/rejected": -11.76340389251709,
+      "loss": 0.4521,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.264101028442383,
+      "rewards/margins": 1.4993038177490234,
+      "rewards/rejected": -11.76340389251709,
+      "semantic_entropy": 0.000987940700724721,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 31.53437456564117,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": 1.5709153413772583,
+      "logits/rejected": 1.6195738315582275,
+      "logps/chosen": -10.149914741516113,
+      "logps/rejected": -11.645903587341309,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.149914741516113,
+      "rewards/margins": 1.4959896802902222,
+      "rewards/rejected": -11.645903587341309,
+      "semantic_entropy": 0.0010429846588522196,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 23.4796696683527,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": 1.5418881177902222,
+      "logits/rejected": 1.5981451272964478,
+      "logps/chosen": -9.971334457397461,
+      "logps/rejected": -11.506645202636719,
+      "loss": 0.3921,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.971334457397461,
+      "rewards/margins": 1.535309910774231,
+      "rewards/rejected": -11.506645202636719,
+      "semantic_entropy": 0.00117585773114115,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 17.691575769391978,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": 1.5182737112045288,
+      "logits/rejected": 1.5788276195526123,
+      "logps/chosen": -10.011544227600098,
+      "logps/rejected": -11.597541809082031,
+      "loss": 0.369,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.011544227600098,
+      "rewards/margins": 1.5859973430633545,
+      "rewards/rejected": -11.597541809082031,
+      "semantic_entropy": 0.0011054911883547902,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 27.477795550751125,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": 1.576537013053894,
+      "logits/rejected": 1.6346805095672607,
+      "logps/chosen": -9.954521179199219,
+      "logps/rejected": -11.532373428344727,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.954521179199219,
+      "rewards/margins": 1.5778512954711914,
+      "rewards/rejected": -11.532373428344727,
+      "semantic_entropy": 0.0011960715055465698,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 26.41381017795156,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": 1.6196601390838623,
+      "logits/rejected": 1.7120628356933594,
+      "logps/chosen": -10.298471450805664,
+      "logps/rejected": -11.907098770141602,
+      "loss": 0.341,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.298471450805664,
+      "rewards/margins": 1.6086266040802002,
+      "rewards/rejected": -11.907098770141602,
+      "semantic_entropy": 0.0010750473011285067,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 23.464195417088263,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": 1.5908796787261963,
+      "logits/rejected": 1.668788194656372,
+      "logps/chosen": -10.35992431640625,
+      "logps/rejected": -11.906169891357422,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.35992431640625,
+      "rewards/margins": 1.5462459325790405,
+      "rewards/rejected": -11.906169891357422,
+      "semantic_entropy": 0.0008059014799073339,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 40.10447162122613,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": 1.6088069677352905,
+      "logits/rejected": 1.6220715045928955,
+      "logps/chosen": -10.40576457977295,
+      "logps/rejected": -11.74560546875,
+      "loss": 0.4161,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.40576457977295,
+      "rewards/margins": 1.3398401737213135,
+      "rewards/rejected": -11.74560546875,
+      "semantic_entropy": 0.0008410325972363353,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 21.06814629981367,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": 1.6059916019439697,
+      "logits/rejected": 1.6614573001861572,
+      "logps/chosen": -10.004603385925293,
+      "logps/rejected": -11.718924522399902,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.004603385925293,
+      "rewards/margins": 1.7143211364746094,
+      "rewards/rejected": -11.718924522399902,
+      "semantic_entropy": 0.0017309269169345498,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 30.19801044462313,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": 1.532857060432434,
+      "logits/rejected": 1.6101768016815186,
+      "logps/chosen": -10.11087703704834,
+      "logps/rejected": -11.715932846069336,
+      "loss": 0.3583,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.11087703704834,
+      "rewards/margins": 1.6050548553466797,
+      "rewards/rejected": -11.715932846069336,
+      "semantic_entropy": 0.001267996965907514,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 26.385907563230248,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": 1.6328445672988892,
+      "logits/rejected": 1.7106590270996094,
+      "logps/chosen": -10.104466438293457,
+      "logps/rejected": -11.775286674499512,
+      "loss": 0.3467,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.104466438293457,
+      "rewards/margins": 1.6708208322525024,
+      "rewards/rejected": -11.775286674499512,
+      "semantic_entropy": 0.0011675632558763027,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 34.83173903448878,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": 1.5373525619506836,
+      "logits/rejected": 1.6098172664642334,
+      "logps/chosen": -10.244033813476562,
+      "logps/rejected": -11.658476829528809,
+      "loss": 0.4342,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.244033813476562,
+      "rewards/margins": 1.4144436120986938,
+      "rewards/rejected": -11.658476829528809,
+      "semantic_entropy": 0.0008294666185975075,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": 1.6723942756652832,
+      "eval_logits/rejected": 1.71475350856781,
+      "eval_logps/chosen": -10.399833679199219,
+      "eval_logps/rejected": -11.605624198913574,
+      "eval_loss": 0.5441703796386719,
+      "eval_rewards/accuracies": 0.7225519418716431,
+      "eval_rewards/chosen": -10.399833679199219,
+      "eval_rewards/margins": 1.205790400505066,
+      "eval_rewards/rejected": -11.605624198913574,
+      "eval_runtime": 34.7924,
+      "eval_samples_per_second": 38.658,
+      "eval_semantic_entropy": 0.0009846173925325274,
+      "eval_steps_per_second": 9.686,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 19.130396062629767,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": 1.6051982641220093,
+      "logits/rejected": 1.6293089389801025,
+      "logps/chosen": -10.201861381530762,
+      "logps/rejected": -11.747920989990234,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.201861381530762,
+      "rewards/margins": 1.5460599660873413,
+      "rewards/rejected": -11.747920989990234,
+      "semantic_entropy": 0.0011727616656571627,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 26.765566782791648,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": 1.610408067703247,
+      "logits/rejected": 1.6632477045059204,
+      "logps/chosen": -10.2145414352417,
+      "logps/rejected": -11.832659721374512,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.2145414352417,
+      "rewards/margins": 1.6181180477142334,
+      "rewards/rejected": -11.832659721374512,
+      "semantic_entropy": 0.0009072507964447141,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 24.393104189390627,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": 1.6049264669418335,
+      "logits/rejected": 1.665951132774353,
+      "logps/chosen": -10.027270317077637,
+      "logps/rejected": -11.678857803344727,
+      "loss": 0.3477,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.027270317077637,
+      "rewards/margins": 1.6515880823135376,
+      "rewards/rejected": -11.678857803344727,
+      "semantic_entropy": 0.0011158749694004655,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 26.27225781883426,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": 1.542015790939331,
+      "logits/rejected": 1.6123573780059814,
+      "logps/chosen": -10.12824535369873,
+      "logps/rejected": -11.875028610229492,
+      "loss": 0.3253,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.12824535369873,
+      "rewards/margins": 1.746783971786499,
+      "rewards/rejected": -11.875028610229492,
+      "semantic_entropy": 0.0009597282623872161,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 32.95954625681789,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": 1.5655219554901123,
+      "logits/rejected": 1.6554622650146484,
+      "logps/chosen": -10.121625900268555,
+      "logps/rejected": -11.683788299560547,
+      "loss": 0.375,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.121625900268555,
+      "rewards/margins": 1.5621623992919922,
+      "rewards/rejected": -11.683788299560547,
+      "semantic_entropy": 0.0010449540568515658,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 16.291440828442056,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": 1.529916524887085,
+      "logits/rejected": 1.6003299951553345,
+      "logps/chosen": -10.08607292175293,
+      "logps/rejected": -11.704909324645996,
+      "loss": 0.3437,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.08607292175293,
+      "rewards/margins": 1.6188377141952515,
+      "rewards/rejected": -11.704909324645996,
+      "semantic_entropy": 0.0010451633716002107,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 26.387562826362483,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": 1.5168224573135376,
+      "logits/rejected": 1.584348440170288,
+      "logps/chosen": -9.997157096862793,
+      "logps/rejected": -11.565196990966797,
+      "loss": 0.3755,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.997157096862793,
+      "rewards/margins": 1.568039059638977,
+      "rewards/rejected": -11.565196990966797,
+      "semantic_entropy": 0.0010721833677962422,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 25.680973380813125,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": 1.5804258584976196,
+      "logits/rejected": 1.6510130167007446,
+      "logps/chosen": -10.183052062988281,
+      "logps/rejected": -11.685959815979004,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.183052062988281,
+      "rewards/margins": 1.5029072761535645,
+      "rewards/rejected": -11.685959815979004,
+      "semantic_entropy": 0.0010565028060227633,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 27.863978397508642,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": 1.6222890615463257,
+      "logits/rejected": 1.654661774635315,
+      "logps/chosen": -10.223709106445312,
+      "logps/rejected": -11.596976280212402,
+      "loss": 0.4066,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.223709106445312,
+      "rewards/margins": 1.373267412185669,
+      "rewards/rejected": -11.596976280212402,
+      "semantic_entropy": 0.0009394630906172097,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 34.93214182012823,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": 1.5334242582321167,
+      "logits/rejected": 1.6439768075942993,
+      "logps/chosen": -10.221205711364746,
+      "logps/rejected": -11.774901390075684,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.221205711364746,
+      "rewards/margins": 1.553695797920227,
+      "rewards/rejected": -11.774901390075684,
+      "semantic_entropy": 0.0011233913246542215,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 21.936988568782077,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": 1.577123761177063,
+      "logits/rejected": 1.6522834300994873,
+      "logps/chosen": -10.454652786254883,
+      "logps/rejected": -11.985879898071289,
+      "loss": 0.3522,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.454652786254883,
+      "rewards/margins": 1.531226396560669,
+      "rewards/rejected": -11.985879898071289,
+      "semantic_entropy": 0.0008189602522179484,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 53.92141622938354,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": 1.578498125076294,
+      "logits/rejected": 1.6103458404541016,
+      "logps/chosen": -10.18836784362793,
+      "logps/rejected": -11.612798690795898,
+      "loss": 0.4441,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.18836784362793,
+      "rewards/margins": 1.4244304895401,
+      "rewards/rejected": -11.612798690795898,
+      "semantic_entropy": 0.001036794506944716,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 23.440976582392643,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": 1.561290979385376,
+      "logits/rejected": 1.6258106231689453,
+      "logps/chosen": -10.179496765136719,
+      "logps/rejected": -11.716876029968262,
+      "loss": 0.3584,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.179496765136719,
+      "rewards/margins": 1.5373791456222534,
+      "rewards/rejected": -11.716876029968262,
+      "semantic_entropy": 0.0010474406881257892,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 32.295062416652954,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": 1.6407121419906616,
+      "logits/rejected": 1.6883310079574585,
+      "logps/chosen": -10.212575912475586,
+      "logps/rejected": -11.806300163269043,
+      "loss": 0.3641,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.212575912475586,
+      "rewards/margins": 1.5937249660491943,
+      "rewards/rejected": -11.806300163269043,
+      "semantic_entropy": 0.0011026955908164382,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 19.731382249910407,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": 1.5847362279891968,
+      "logits/rejected": 1.6605494022369385,
+      "logps/chosen": -10.028234481811523,
+      "logps/rejected": -11.671747207641602,
+      "loss": 0.3217,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.028234481811523,
+      "rewards/margins": 1.643511176109314,
+      "rewards/rejected": -11.671747207641602,
+      "semantic_entropy": 0.0011805352987721562,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 24.89822301014984,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": 1.5898643732070923,
+      "logits/rejected": 1.6592276096343994,
+      "logps/chosen": -10.474271774291992,
+      "logps/rejected": -12.109563827514648,
+      "loss": 0.3548,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.474271774291992,
+      "rewards/margins": 1.635292649269104,
+      "rewards/rejected": -12.109563827514648,
+      "semantic_entropy": 0.0012867021141573787,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 24.45818990722533,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": 1.5716315507888794,
+      "logits/rejected": 1.643298864364624,
+      "logps/chosen": -10.188231468200684,
+      "logps/rejected": -11.930964469909668,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.188231468200684,
+      "rewards/margins": 1.7427337169647217,
+      "rewards/rejected": -11.930964469909668,
+      "semantic_entropy": 0.0010645693400874734,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 32.717354981867146,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": 1.5848592519760132,
+      "logits/rejected": 1.6642652750015259,
+      "logps/chosen": -10.201481819152832,
+      "logps/rejected": -11.77349853515625,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.201481819152832,
+      "rewards/margins": 1.5720161199569702,
+      "rewards/rejected": -11.77349853515625,
+      "semantic_entropy": 0.001312751672230661,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 22.12852451685905,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": 1.5790159702301025,
+      "logits/rejected": 1.6645339727401733,
+      "logps/chosen": -10.404925346374512,
+      "logps/rejected": -11.937236785888672,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.404925346374512,
+      "rewards/margins": 1.532311201095581,
+      "rewards/rejected": -11.937236785888672,
+      "semantic_entropy": 0.000721759395673871,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 25.93019243165944,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": 1.5529661178588867,
+      "logits/rejected": 1.6045109033584595,
+      "logps/chosen": -10.146554946899414,
+      "logps/rejected": -11.785726547241211,
+      "loss": 0.3488,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.146554946899414,
+      "rewards/margins": 1.6391710042953491,
+      "rewards/rejected": -11.785726547241211,
+      "semantic_entropy": 0.00114902772475034,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 33.49006304599703,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": 1.6349928379058838,
+      "logits/rejected": 1.6662530899047852,
+      "logps/chosen": -10.314465522766113,
+      "logps/rejected": -11.75831127166748,
+      "loss": 0.4262,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.314465522766113,
+      "rewards/margins": 1.4438472986221313,
+      "rewards/rejected": -11.75831127166748,
+      "semantic_entropy": 0.0008874175255186856,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 24.701056923944595,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": 1.5712239742279053,
+      "logits/rejected": 1.650539755821228,
+      "logps/chosen": -10.23160457611084,
+      "logps/rejected": -12.1198148727417,
+      "loss": 0.3626,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -10.23160457611084,
+      "rewards/margins": 1.8882091045379639,
+      "rewards/rejected": -12.1198148727417,
+      "semantic_entropy": 0.000983731122687459,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 30.114634789412605,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": 1.6887805461883545,
+      "logits/rejected": 1.7742246389389038,
+      "logps/chosen": -10.461101531982422,
+      "logps/rejected": -11.841108322143555,
+      "loss": 0.4105,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.461101531982422,
+      "rewards/margins": 1.3800055980682373,
+      "rewards/rejected": -11.841108322143555,
+      "semantic_entropy": 0.0008623698959127069,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 30.016876238895843,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": 1.6389271020889282,
+      "logits/rejected": 1.681368112564087,
+      "logps/chosen": -10.227550506591797,
+      "logps/rejected": -11.788646697998047,
+      "loss": 0.3499,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.227550506591797,
+      "rewards/margins": 1.56109619140625,
+      "rewards/rejected": -11.788646697998047,
+      "semantic_entropy": 0.000960902136284858,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 24.81310196445911,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": 1.60993230342865,
+      "logits/rejected": 1.694504737854004,
+      "logps/chosen": -10.357182502746582,
+      "logps/rejected": -11.963488578796387,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.357182502746582,
+      "rewards/margins": 1.6063076257705688,
+      "rewards/rejected": -11.963488578796387,
+      "semantic_entropy": 0.0009183714864775538,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 31.734919941247338,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": 1.6071914434432983,
+      "logits/rejected": 1.6563193798065186,
+      "logps/chosen": -10.27714729309082,
+      "logps/rejected": -11.882461547851562,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -10.27714729309082,
+      "rewards/margins": 1.6053130626678467,
+      "rewards/rejected": -11.882461547851562,
+      "semantic_entropy": 0.0011328940745443106,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 27.79315590237926,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": 1.5941672325134277,
+      "logits/rejected": 1.613385558128357,
+      "logps/chosen": -10.245985984802246,
+      "logps/rejected": -11.532986640930176,
+      "loss": 0.4582,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.245985984802246,
+      "rewards/margins": 1.2870004177093506,
+      "rewards/rejected": -11.532986640930176,
+      "semantic_entropy": 0.0010124377440661192,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 24.411714875135623,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": 1.5859867334365845,
+      "logits/rejected": 1.6503912210464478,
+      "logps/chosen": -10.193082809448242,
+      "logps/rejected": -11.835702896118164,
+      "loss": 0.3629,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.193082809448242,
+      "rewards/margins": 1.6426210403442383,
+      "rewards/rejected": -11.835702896118164,
+      "semantic_entropy": 0.0010750694200396538,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 26.14103588408639,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": 1.5207525491714478,
+      "logits/rejected": 1.5767511129379272,
+      "logps/chosen": -10.251093864440918,
+      "logps/rejected": -11.818537712097168,
+      "loss": 0.335,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.251093864440918,
+      "rewards/margins": 1.5674444437026978,
+      "rewards/rejected": -11.818537712097168,
+      "semantic_entropy": 0.0009068836225196719,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 27.654754214457785,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": 1.5871937274932861,
+      "logits/rejected": 1.6162084341049194,
+      "logps/chosen": -10.149468421936035,
+      "logps/rejected": -11.509541511535645,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.149468421936035,
+      "rewards/margins": 1.3600738048553467,
+      "rewards/rejected": -11.509541511535645,
+      "semantic_entropy": 0.0012269694125279784,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 17.914085105445775,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": 1.6091039180755615,
+      "logits/rejected": 1.6779155731201172,
+      "logps/chosen": -10.301715850830078,
+      "logps/rejected": -11.755305290222168,
+      "loss": 0.3861,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.301715850830078,
+      "rewards/margins": 1.4535901546478271,
+      "rewards/rejected": -11.755305290222168,
+      "semantic_entropy": 0.0009987682569772005,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 33.221478802826475,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": 1.5723199844360352,
+      "logits/rejected": 1.6276495456695557,
+      "logps/chosen": -10.17293930053711,
+      "logps/rejected": -11.548049926757812,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.17293930053711,
+      "rewards/margins": 1.3751102685928345,
+      "rewards/rejected": -11.548049926757812,
+      "semantic_entropy": 0.001041922951117158,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 19.886426451357405,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": 1.5793964862823486,
+      "logits/rejected": 1.6333627700805664,
+      "logps/chosen": -10.343340873718262,
+      "logps/rejected": -11.866167068481445,
+      "loss": 0.4031,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.343340873718262,
+      "rewards/margins": 1.5228270292282104,
+      "rewards/rejected": -11.866167068481445,
+      "semantic_entropy": 0.0008631997625343502,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 26.788395511633848,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": 1.6743360757827759,
+      "logits/rejected": 1.7168939113616943,
+      "logps/chosen": -10.264796257019043,
+      "logps/rejected": -11.578871726989746,
+      "loss": 0.4351,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.264796257019043,
+      "rewards/margins": 1.314075231552124,
+      "rewards/rejected": -11.578871726989746,
+      "semantic_entropy": 0.0010403693886473775,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 30.38243334684144,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": 1.6089446544647217,
+      "logits/rejected": 1.6752182245254517,
+      "logps/chosen": -9.972900390625,
+      "logps/rejected": -11.780462265014648,
+      "loss": 0.3579,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -9.972900390625,
+      "rewards/margins": 1.8075612783432007,
+      "rewards/rejected": -11.780462265014648,
+      "semantic_entropy": 0.0011388263665139675,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 23.66108212189689,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": 1.5644620656967163,
+      "logits/rejected": 1.6685336828231812,
+      "logps/chosen": -10.10273551940918,
+      "logps/rejected": -11.810322761535645,
+      "loss": 0.3191,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.10273551940918,
+      "rewards/margins": 1.707586646080017,
+      "rewards/rejected": -11.810322761535645,
+      "semantic_entropy": 0.000993860885500908,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 19.038339346614674,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": 1.640947699546814,
+      "logits/rejected": 1.689212441444397,
+      "logps/chosen": -10.065169334411621,
+      "logps/rejected": -11.680166244506836,
+      "loss": 0.3561,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.065169334411621,
+      "rewards/margins": 1.614996314048767,
+      "rewards/rejected": -11.680166244506836,
+      "semantic_entropy": 0.001320890849456191,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 29.17177091523795,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": 1.6104345321655273,
+      "logits/rejected": 1.6885284185409546,
+      "logps/chosen": -10.547137260437012,
+      "logps/rejected": -11.81525707244873,
+      "loss": 0.4377,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -10.547137260437012,
+      "rewards/margins": 1.268120288848877,
+      "rewards/rejected": -11.81525707244873,
+      "semantic_entropy": 0.0009555103024467826,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 30.119756881610734,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": 1.4851740598678589,
+      "logits/rejected": 1.5895230770111084,
+      "logps/chosen": -10.068452835083008,
+      "logps/rejected": -11.538588523864746,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.068452835083008,
+      "rewards/margins": 1.4701359272003174,
+      "rewards/rejected": -11.538588523864746,
+      "semantic_entropy": 0.0011919718235731125,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 24.55816949250339,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": 1.5584100484848022,
+      "logits/rejected": 1.6823928356170654,
+      "logps/chosen": -10.034466743469238,
+      "logps/rejected": -11.935209274291992,
+      "loss": 0.339,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.034466743469238,
+      "rewards/margins": 1.9007441997528076,
+      "rewards/rejected": -11.935209274291992,
+      "semantic_entropy": 0.0012700663646683097,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 23.62435769803573,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": 1.5819255113601685,
+      "logits/rejected": 1.6369807720184326,
+      "logps/chosen": -10.297229766845703,
+      "logps/rejected": -11.85633373260498,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.297229766845703,
+      "rewards/margins": 1.5591046810150146,
+      "rewards/rejected": -11.85633373260498,
+      "semantic_entropy": 0.0011318308534100652,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 18.948644831188616,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": 1.6146568059921265,
+      "logits/rejected": 1.6612657308578491,
+      "logps/chosen": -10.349318504333496,
+      "logps/rejected": -11.954682350158691,
+      "loss": 0.412,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.349318504333496,
+      "rewards/margins": 1.6053638458251953,
+      "rewards/rejected": -11.954682350158691,
+      "semantic_entropy": 0.0008482816629111767,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 21.069776978148884,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": 1.5913143157958984,
+      "logits/rejected": 1.6983072757720947,
+      "logps/chosen": -10.35401725769043,
+      "logps/rejected": -11.782625198364258,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.35401725769043,
+      "rewards/margins": 1.428606629371643,
+      "rewards/rejected": -11.782625198364258,
+      "semantic_entropy": 0.0008745426312088966,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 24.086974521849672,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": 1.6169697046279907,
+      "logits/rejected": 1.6588325500488281,
+      "logps/chosen": -10.274133682250977,
+      "logps/rejected": -11.636290550231934,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.274133682250977,
+      "rewards/margins": 1.3621585369110107,
+      "rewards/rejected": -11.636290550231934,
+      "semantic_entropy": 0.0008288308745250106,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 33.17375750283582,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": 1.5760698318481445,
+      "logits/rejected": 1.6068201065063477,
+      "logps/chosen": -10.287278175354004,
+      "logps/rejected": -11.950657844543457,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.287278175354004,
+      "rewards/margins": 1.6633796691894531,
+      "rewards/rejected": -11.950657844543457,
+      "semantic_entropy": 0.0009137185988947749,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 24.965105374392788,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": 1.55120050907135,
+      "logits/rejected": 1.6359049081802368,
+      "logps/chosen": -10.309528350830078,
+      "logps/rejected": -12.006683349609375,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.309528350830078,
+      "rewards/margins": 1.6971546411514282,
+      "rewards/rejected": -12.006683349609375,
+      "semantic_entropy": 0.0025044563226401806,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 28.087049375489126,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": 1.5590399503707886,
+      "logits/rejected": 1.6054328680038452,
+      "logps/chosen": -10.253583908081055,
+      "logps/rejected": -11.814079284667969,
+      "loss": 0.3584,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.253583908081055,
+      "rewards/margins": 1.560494065284729,
+      "rewards/rejected": -11.814079284667969,
+      "semantic_entropy": 0.0010741263395175338,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 28.3180971763924,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": 1.5820307731628418,
+      "logits/rejected": 1.648240327835083,
+      "logps/chosen": -10.3207426071167,
+      "logps/rejected": -11.669373512268066,
+      "loss": 0.4267,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.3207426071167,
+      "rewards/margins": 1.348630666732788,
+      "rewards/rejected": -11.669373512268066,
+      "semantic_entropy": 0.0008240357274189591,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 35.007764300947755,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": 1.6360353231430054,
+      "logits/rejected": 1.6682937145233154,
+      "logps/chosen": -10.0896577835083,
+      "logps/rejected": -11.733640670776367,
+      "loss": 0.4642,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -10.0896577835083,
+      "rewards/margins": 1.6439838409423828,
+      "rewards/rejected": -11.733640670776367,
+      "semantic_entropy": 0.0011510265758261085,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 31.6773523831241,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": 1.641503095626831,
+      "logits/rejected": 1.6899493932724,
+      "logps/chosen": -10.28085994720459,
+      "logps/rejected": -11.787833213806152,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.28085994720459,
+      "rewards/margins": 1.5069721937179565,
+      "rewards/rejected": -11.787833213806152,
+      "semantic_entropy": 0.0009235903853550553,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 25.58516059271576,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": 1.524792194366455,
+      "logits/rejected": 1.6251424551010132,
+      "logps/chosen": -10.28318977355957,
+      "logps/rejected": -11.53878402709961,
+      "loss": 0.4424,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.28318977355957,
+      "rewards/margins": 1.2555944919586182,
+      "rewards/rejected": -11.53878402709961,
+      "semantic_entropy": 0.0008631746168248355,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 22.321837351423174,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": 1.5386344194412231,
+      "logits/rejected": 1.6239769458770752,
+      "logps/chosen": -9.959778785705566,
+      "logps/rejected": -11.631715774536133,
+      "loss": 0.3753,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.959778785705566,
+      "rewards/margins": 1.6719348430633545,
+      "rewards/rejected": -11.631715774536133,
+      "semantic_entropy": 0.0011183499591425061,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 24.13182522047276,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": 1.545141577720642,
+      "logits/rejected": 1.6213299036026,
+      "logps/chosen": -10.363487243652344,
+      "logps/rejected": -11.634809494018555,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -10.363487243652344,
+      "rewards/margins": 1.271321177482605,
+      "rewards/rejected": -11.634809494018555,
+      "semantic_entropy": 0.0008362028747797012,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 39.442905774033456,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": 1.5921802520751953,
+      "logits/rejected": 1.6468632221221924,
+      "logps/chosen": -10.312657356262207,
+      "logps/rejected": -11.642027854919434,
+      "loss": 0.4467,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.312657356262207,
+      "rewards/margins": 1.3293715715408325,
+      "rewards/rejected": -11.642027854919434,
+      "semantic_entropy": 0.0009145711665041745,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 28.932958200452326,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": 1.6327731609344482,
+      "logits/rejected": 1.634010672569275,
+      "logps/chosen": -10.188440322875977,
+      "logps/rejected": -11.656460762023926,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.188440322875977,
+      "rewards/margins": 1.4680200815200806,
+      "rewards/rejected": -11.656460762023926,
+      "semantic_entropy": 0.0011566232424229383,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 24.187636967913257,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": 1.5645967721939087,
+      "logits/rejected": 1.6284879446029663,
+      "logps/chosen": -10.217806816101074,
+      "logps/rejected": -11.735448837280273,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.217806816101074,
+      "rewards/margins": 1.5176423788070679,
+      "rewards/rejected": -11.735448837280273,
+      "semantic_entropy": 0.0010513458400964737,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 25.605827763062255,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": 1.5997997522354126,
+      "logits/rejected": 1.6790711879730225,
+      "logps/chosen": -10.027923583984375,
+      "logps/rejected": -11.390928268432617,
+      "loss": 0.391,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.027923583984375,
+      "rewards/margins": 1.3630036115646362,
+      "rewards/rejected": -11.390928268432617,
+      "semantic_entropy": 0.0012856271350756288,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 33.99450540894745,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": 1.5260295867919922,
+      "logits/rejected": 1.5822535753250122,
+      "logps/chosen": -10.097551345825195,
+      "logps/rejected": -11.618727684020996,
+      "loss": 0.3915,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.097551345825195,
+      "rewards/margins": 1.5211765766143799,
+      "rewards/rejected": -11.618727684020996,
+      "semantic_entropy": 0.0010385962668806314,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 18.344410764482195,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": 1.5907096862792969,
+      "logits/rejected": 1.6714776754379272,
+      "logps/chosen": -10.2022705078125,
+      "logps/rejected": -11.915973663330078,
+      "loss": 0.3527,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -10.2022705078125,
+      "rewards/margins": 1.713702917098999,
+      "rewards/rejected": -11.915973663330078,
+      "semantic_entropy": 0.0008863688562996686,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 22.485876723124694,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": 1.628910779953003,
+      "logits/rejected": 1.6816396713256836,
+      "logps/chosen": -10.326075553894043,
+      "logps/rejected": -11.664091110229492,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -10.326075553894043,
+      "rewards/margins": 1.3380156755447388,
+      "rewards/rejected": -11.664091110229492,
+      "semantic_entropy": 0.0010224630823358893,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 30.668572637430486,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": 1.5799617767333984,
+      "logits/rejected": 1.60642409324646,
+      "logps/chosen": -10.205876350402832,
+      "logps/rejected": -11.643888473510742,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.205876350402832,
+      "rewards/margins": 1.4380121231079102,
+      "rewards/rejected": -11.643888473510742,
+      "semantic_entropy": 0.001081346534192562,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 26.914987261125045,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": 1.5873939990997314,
+      "logits/rejected": 1.673717737197876,
+      "logps/chosen": -10.314226150512695,
+      "logps/rejected": -12.065945625305176,
+      "loss": 0.351,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.314226150512695,
+      "rewards/margins": 1.7517201900482178,
+      "rewards/rejected": -12.065945625305176,
+      "semantic_entropy": 0.000804221723228693,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 32.249711323613845,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": 1.5093019008636475,
+      "logits/rejected": 1.570624589920044,
+      "logps/chosen": -10.205594062805176,
+      "logps/rejected": -11.78739070892334,
+      "loss": 0.3812,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.205594062805176,
+      "rewards/margins": 1.581796646118164,
+      "rewards/rejected": -11.78739070892334,
+      "semantic_entropy": 0.0009667190606705844,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 26.73308539502373,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": 1.5923975706100464,
+      "logits/rejected": 1.6267331838607788,
+      "logps/chosen": -10.20487117767334,
+      "logps/rejected": -11.733766555786133,
+      "loss": 0.3604,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.20487117767334,
+      "rewards/margins": 1.5288949012756348,
+      "rewards/rejected": -11.733766555786133,
+      "semantic_entropy": 0.0011182560119777918,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 25.81036525979864,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": 1.5881580114364624,
+      "logits/rejected": 1.681006669998169,
+      "logps/chosen": -10.442743301391602,
+      "logps/rejected": -11.997671127319336,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.442743301391602,
+      "rewards/margins": 1.554928183555603,
+      "rewards/rejected": -11.997671127319336,
+      "semantic_entropy": 0.0007110863225534558,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 23.03542466897119,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": 1.532167911529541,
+      "logits/rejected": 1.6093155145645142,
+      "logps/chosen": -10.341131210327148,
+      "logps/rejected": -11.707452774047852,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.341131210327148,
+      "rewards/margins": 1.3663222789764404,
+      "rewards/rejected": -11.707452774047852,
+      "semantic_entropy": 0.0008603068999946117,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 32.94534776202207,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": 1.5275776386260986,
+      "logits/rejected": 1.5687650442123413,
+      "logps/chosen": -10.387439727783203,
+      "logps/rejected": -11.788525581359863,
+      "loss": 0.3994,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.387439727783203,
+      "rewards/margins": 1.4010872840881348,
+      "rewards/rejected": -11.788525581359863,
+      "semantic_entropy": 0.0008422583341598511,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 33.3881008372957,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": 1.5168180465698242,
+      "logits/rejected": 1.5796856880187988,
+      "logps/chosen": -10.338147163391113,
+      "logps/rejected": -11.821488380432129,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.338147163391113,
+      "rewards/margins": 1.483341932296753,
+      "rewards/rejected": -11.821488380432129,
+      "semantic_entropy": 0.0007745058974251151,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 23.55740151197903,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": 1.5829195976257324,
+      "logits/rejected": 1.622502088546753,
+      "logps/chosen": -10.269952774047852,
+      "logps/rejected": -11.729962348937988,
+      "loss": 0.397,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -10.269952774047852,
+      "rewards/margins": 1.4600093364715576,
+      "rewards/rejected": -11.729962348937988,
+      "semantic_entropy": 0.0008267055382020772,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 17.603422872872386,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": 1.6074835062026978,
+      "logits/rejected": 1.6517820358276367,
+      "logps/chosen": -10.132599830627441,
+      "logps/rejected": -11.54753303527832,
+      "loss": 0.3659,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.132599830627441,
+      "rewards/margins": 1.4149324893951416,
+      "rewards/rejected": -11.54753303527832,
+      "semantic_entropy": 0.000982355559244752,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 29.389136403221865,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": 1.546082854270935,
+      "logits/rejected": 1.6363255977630615,
+      "logps/chosen": -10.170438766479492,
+      "logps/rejected": -11.964184761047363,
+      "loss": 0.3337,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.170438766479492,
+      "rewards/margins": 1.7937456369400024,
+      "rewards/rejected": -11.964184761047363,
+      "semantic_entropy": 0.0009365129517391324,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 26.299431962432394,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": 1.5456407070159912,
+      "logits/rejected": 1.594358205795288,
+      "logps/chosen": -10.266721725463867,
+      "logps/rejected": -11.552698135375977,
+      "loss": 0.4373,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -10.266721725463867,
+      "rewards/margins": 1.285976767539978,
+      "rewards/rejected": -11.552698135375977,
+      "semantic_entropy": 0.0008851070888340473,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 26.132191696338683,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": 1.563470482826233,
+      "logits/rejected": 1.6354659795761108,
+      "logps/chosen": -10.240187644958496,
+      "logps/rejected": -11.771659851074219,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.240187644958496,
+      "rewards/margins": 1.531471610069275,
+      "rewards/rejected": -11.771659851074219,
+      "semantic_entropy": 0.0008821386145427823,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 23.75062796381459,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": 1.5944634675979614,
+      "logits/rejected": 1.6611665487289429,
+      "logps/chosen": -10.245941162109375,
+      "logps/rejected": -11.986920356750488,
+      "loss": 0.3661,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -10.245941162109375,
+      "rewards/margins": 1.7409788370132446,
+      "rewards/rejected": -11.986920356750488,
+      "semantic_entropy": 0.0010045578237622976,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 25.372210771581525,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": 1.546979546546936,
+      "logits/rejected": 1.6090978384017944,
+      "logps/chosen": -9.878134727478027,
+      "logps/rejected": -11.502184867858887,
+      "loss": 0.3853,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -9.878134727478027,
+      "rewards/margins": 1.6240507364273071,
+      "rewards/rejected": -11.502184867858887,
+      "semantic_entropy": 0.0011057776864618063,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 34.33094094457402,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": 1.5365062952041626,
+      "logits/rejected": 1.6328754425048828,
+      "logps/chosen": -10.134115219116211,
+      "logps/rejected": -11.8469820022583,
+      "loss": 0.368,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.134115219116211,
+      "rewards/margins": 1.7128671407699585,
+      "rewards/rejected": -11.8469820022583,
+      "semantic_entropy": 0.0013317258562892675,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 16.42051300346504,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": 1.493348240852356,
+      "logits/rejected": 1.573560357093811,
+      "logps/chosen": -10.147087097167969,
+      "logps/rejected": -11.685896873474121,
+      "loss": 0.3638,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -10.147087097167969,
+      "rewards/margins": 1.538810133934021,
+      "rewards/rejected": -11.685896873474121,
+      "semantic_entropy": 0.0011134275700896978,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 18.113716326941372,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": 1.5595808029174805,
+      "logits/rejected": 1.6234403848648071,
+      "logps/chosen": -10.314157485961914,
+      "logps/rejected": -11.681783676147461,
+      "loss": 0.4015,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -10.314157485961914,
+      "rewards/margins": 1.3676280975341797,
+      "rewards/rejected": -11.681783676147461,
+      "semantic_entropy": 0.0009090052917599678,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 33.814971635899745,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": 1.5722408294677734,
+      "logits/rejected": 1.6538846492767334,
+      "logps/chosen": -10.236674308776855,
+      "logps/rejected": -11.754487991333008,
+      "loss": 0.394,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -10.236674308776855,
+      "rewards/margins": 1.517815351486206,
+      "rewards/rejected": -11.754487991333008,
+      "semantic_entropy": 0.001014344277791679,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 29.252897521112676,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": 1.573158621788025,
+      "logits/rejected": 1.6159855127334595,
+      "logps/chosen": -10.092769622802734,
+      "logps/rejected": -11.823246002197266,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -10.092769622802734,
+      "rewards/margins": 1.7304775714874268,
+      "rewards/rejected": -11.823246002197266,
+      "semantic_entropy": 0.0013433097628876567,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": 1.659920573234558,
+      "eval_logits/rejected": 1.7005716562271118,
+      "eval_logps/chosen": -10.457354545593262,
+      "eval_logps/rejected": -11.66568374633789,
+      "eval_loss": 0.5442783236503601,
+      "eval_rewards/accuracies": 0.721068263053894,
+      "eval_rewards/chosen": -10.457354545593262,
+      "eval_rewards/margins": 1.2083282470703125,
+      "eval_rewards/rejected": -11.66568374633789,
+      "eval_runtime": 34.5418,
+      "eval_samples_per_second": 38.938,
+      "eval_semantic_entropy": 0.0009483197354711592,
+      "eval_steps_per_second": 9.756,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 0.541753981451236,
+      "train_runtime": 28781.2396,
+      "train_samples_per_second": 6.232,
+      "train_steps_per_second": 0.195
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}