Upload folder using huggingface_hub

.gitattributes

@@ -64,3 +64,4 @@ nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024/hf/tokenizer.json filter=
 nemotron_synthetic_1T_exp_7B/metrics.jsonl filter=lfs diff=lfs merge=lfs -text
 nemotron_synthetic_1T_exp_7B/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+nemotron_synthetic_1T_exp_7B_DPO/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text

nemotron_synthetic_1T_exp_7B_DPO/hf/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

nemotron_synthetic_1T_exp_7B_DPO/hf/README.md

@@ -0,0 +1,68 @@
+---
+library_name: transformers
+model_name: nemotron-synthetic-7b-dpo
+tags:
+- generated_from_trainer
+- dpo
+- trl
+licence: license
+---
+
+# Model Card for nemotron-synthetic-7b-dpo
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="craffel/nemotron-synthetic-7b-dpo", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.25.0
+- Transformers: 4.57.1
+- Pytorch: 2.6.0+cu126
+- Datasets: 4.4.1
+- Tokenizers: 0.22.1
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

nemotron_synthetic_1T_exp_7B_DPO/hf/all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -3.4791667461395264,
+    "eval_logits/rejected": -3.5466270446777344,
+    "eval_logps/chosen": -295.77777099609375,
+    "eval_logps/rejected": -278.6825256347656,
+    "eval_loss": 0.6436892151832581,
+    "eval_rewards/accuracies": 0.65625,
+    "eval_rewards/chosen": 1.0559896230697632,
+    "eval_rewards/margins": 1.2326388359069824,
+    "eval_rewards/rejected": -0.1767655611038208,
+    "eval_runtime": 38.0946,
+    "eval_samples_per_second": 52.501,
+    "eval_steps_per_second": 1.654,
+    "total_flos": 0.0,
+    "train_loss": 0.31465955367480697,
+    "train_runtime": 10878.0316,
+    "train_samples": 61135,
+    "train_samples_per_second": 16.86,
+    "train_steps_per_second": 0.132
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/chat_template.jinja

@@ -0,0 +1 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for msg in messages %}{% if msg.role=='user' %}{% if loop.index > 1 %}{{ '\n\n' }}{% endif %}Instruction: {{ msg.content }}{% elif msg.role=='assistant' %}{{ '\n\nAnswer:' }}{{ msg.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '\n\nAnswer:' }}{% endif %}

nemotron_synthetic_1T_exp_7B_DPO/hf/config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 100000.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/eval_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -3.4791667461395264,
+    "eval_logits/rejected": -3.5466270446777344,
+    "eval_logps/chosen": -295.77777099609375,
+    "eval_logps/rejected": -278.6825256347656,
+    "eval_loss": 0.6436892151832581,
+    "eval_rewards/accuracies": 0.65625,
+    "eval_rewards/chosen": 1.0559896230697632,
+    "eval_rewards/margins": 1.2326388359069824,
+    "eval_rewards/rejected": -0.1767655611038208,
+    "eval_runtime": 38.0946,
+    "eval_samples_per_second": 52.501,
+    "eval_steps_per_second": 1.654
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.57.1"
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:688bc3a4938e02758faac9c2bff78dcd3f1f3fcd07dd68a142738084892a8a18
+size 4917979000

nemotron_synthetic_1T_exp_7B_DPO/hf/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bca513ca149b98ad390126ecf7b761512f63bba7e132b5059c40c49e09d09cf
+size 4947390880

nemotron_synthetic_1T_exp_7B_DPO/hf/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65682ade5e2ed6bb0013a391cde3ee197fca5d970c65cd998000cc6ad4793e22
+size 4137880232

nemotron_synthetic_1T_exp_7B_DPO/hf/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e36a565da2b792488f4a02ae108bdfecfd8b33aa4a4c1594275251c2ad2a4268
+size 1050673280

nemotron_synthetic_1T_exp_7B_DPO/hf/model.safetensors.index.json

@@ -0,0 +1,299 @@
+{
+  "metadata": {
+    "total_parameters": 266240,
+    "total_size": 15053889536
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/runs/Nov25_17-31-35_ip-26-0-171-230/events.out.tfevents.1764091937.ip-26-0-171-230.1474540.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f66e9b70f17cf509815ccd270a66e33cadaa47f97dd2be27aea9a4fd3892746
+size 115150

nemotron_synthetic_1T_exp_7B_DPO/hf/runs/Nov25_17-31-35_ip-26-0-171-230/events.out.tfevents.1764102853.ip-26-0-171-230.1474540.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dc905c572e4958fd9c08b0905853e106ab57aef4b0ed5110413943296e3f341
+size 828

nemotron_synthetic_1T_exp_7B_DPO/hf/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|end_of_text|>"
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393
+size 17209961

nemotron_synthetic_1T_exp_7B_DPO/hf/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for msg in messages %}{% if msg.role=='user' %}{% if loop.index > 1 %}{{ '\\n\\n' }}{% endif %}Instruction: {{ msg.content }}{% elif msg.role=='assistant' %}{{ '\\n\\nAnswer:' }}{{ msg.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '\\n\\nAnswer:' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 0.0,
+    "train_loss": 0.31465955367480697,
+    "train_runtime": 10878.0316,
+    "train_samples": 61135,
+    "train_samples_per_second": 16.86,
+    "train_steps_per_second": 0.132
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/trainer_state.json

@@ -0,0 +1,2412 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 100,
+  "global_step": 1434,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 48.848377912242356,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": -3.546093702316284,
+      "logits/rejected": -3.6498045921325684,
+      "logps/chosen": -314.4375,
+      "logps/rejected": -283.26251220703125,
+      "loss": 0.7126,
+      "rewards/accuracies": 0.18671874701976776,
+      "rewards/chosen": 0.0142822265625,
+      "rewards/margins": 0.020751953125,
+      "rewards/rejected": -0.0064697265625,
+      "step": 10
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 59.273271949373076,
+      "learning_rate": 1.3194444444444444e-07,
+      "logits/chosen": -3.533398389816284,
+      "logits/rejected": -3.619335889816284,
+      "logps/chosen": -299.8500061035156,
+      "logps/rejected": -262.8374938964844,
+      "loss": 0.7251,
+      "rewards/accuracies": 0.20703125,
+      "rewards/chosen": -0.007006836123764515,
+      "rewards/margins": -0.007232666015625,
+      "rewards/rejected": 0.00022583008103538305,
+      "step": 20
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 60.22779824387445,
+      "learning_rate": 2.013888888888889e-07,
+      "logits/chosen": -3.5341796875,
+      "logits/rejected": -3.6185545921325684,
+      "logps/chosen": -294.2124938964844,
+      "logps/rejected": -268.6625061035156,
+      "loss": 0.7337,
+      "rewards/accuracies": 0.20000000298023224,
+      "rewards/chosen": -0.0025146484840661287,
+      "rewards/margins": -0.0206298828125,
+      "rewards/rejected": 0.01811523362994194,
+      "step": 30
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 58.90424075515779,
+      "learning_rate": 2.708333333333333e-07,
+      "logits/chosen": -3.562304735183716,
+      "logits/rejected": -3.6949219703674316,
+      "logps/chosen": -308.3374938964844,
+      "logps/rejected": -272.03125,
+      "loss": 0.7278,
+      "rewards/accuracies": 0.20781250298023224,
+      "rewards/chosen": 0.02635497972369194,
+      "rewards/margins": -0.0020507811568677425,
+      "rewards/rejected": 0.02840576134622097,
+      "step": 40
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 57.77495084112175,
+      "learning_rate": 3.402777777777778e-07,
+      "logits/chosen": -3.5130858421325684,
+      "logits/rejected": -3.5555663108825684,
+      "logps/chosen": -311.4624938964844,
+      "logps/rejected": -285.76251220703125,
+      "loss": 0.7192,
+      "rewards/accuracies": 0.23359374701976776,
+      "rewards/chosen": 0.030029296875,
+      "rewards/margins": 0.02037353441119194,
+      "rewards/rejected": 0.009655761532485485,
+      "step": 50
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 51.64856571569684,
+      "learning_rate": 4.0972222222222217e-07,
+      "logits/chosen": -3.5257811546325684,
+      "logits/rejected": -3.656054735183716,
+      "logps/chosen": -324.3125,
+      "logps/rejected": -270.1000061035156,
+      "loss": 0.7018,
+      "rewards/accuracies": 0.2789062559604645,
+      "rewards/chosen": 0.11164550483226776,
+      "rewards/margins": 0.07979736477136612,
+      "rewards/rejected": 0.03187255933880806,
+      "step": 60
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 56.999968016175266,
+      "learning_rate": 4.791666666666667e-07,
+      "logits/chosen": -3.5240235328674316,
+      "logits/rejected": -3.6187500953674316,
+      "logps/chosen": -302.6499938964844,
+      "logps/rejected": -273.4375,
+      "loss": 0.7,
+      "rewards/accuracies": 0.3539062440395355,
+      "rewards/chosen": 0.16118164360523224,
+      "rewards/margins": 0.11168213188648224,
+      "rewards/rejected": 0.04946289211511612,
+      "step": 70
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 55.23572566002827,
+      "learning_rate": 5.486111111111111e-07,
+      "logits/chosen": -3.6148438453674316,
+      "logits/rejected": -3.7431640625,
+      "logps/chosen": -298.89373779296875,
+      "logps/rejected": -266.1875,
+      "loss": 0.7056,
+      "rewards/accuracies": 0.36640626192092896,
+      "rewards/chosen": 0.24953612685203552,
+      "rewards/margins": 0.13271483778953552,
+      "rewards/rejected": 0.11678466945886612,
+      "step": 80
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 53.03897889883259,
+      "learning_rate": 6.180555555555556e-07,
+      "logits/chosen": -3.530468702316284,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -303.45001220703125,
+      "logps/rejected": -259.95623779296875,
+      "loss": 0.6943,
+      "rewards/accuracies": 0.42578125,
+      "rewards/chosen": 0.33854979276657104,
+      "rewards/margins": 0.19456787407398224,
+      "rewards/rejected": 0.14398193359375,
+      "step": 90
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 53.55985177205509,
+      "learning_rate": 6.875e-07,
+      "logits/chosen": -3.5365233421325684,
+      "logits/rejected": -3.674023389816284,
+      "logps/chosen": -312.57501220703125,
+      "logps/rejected": -268.17498779296875,
+      "loss": 0.677,
+      "rewards/accuracies": 0.4671874940395355,
+      "rewards/chosen": 0.4582763612270355,
+      "rewards/margins": 0.27515870332717896,
+      "rewards/rejected": 0.18305663764476776,
+      "step": 100
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "eval_logits/chosen": -3.549851179122925,
+      "eval_logits/rejected": -3.617063522338867,
+      "eval_logps/chosen": -296.8571472167969,
+      "eval_logps/rejected": -277.9047546386719,
+      "eval_loss": 0.7054004073143005,
+      "eval_rewards/accuracies": 0.4781745970249176,
+      "eval_rewards/chosen": 0.4493427574634552,
+      "eval_rewards/margins": 0.217819944024086,
+      "eval_rewards/rejected": 0.2317398339509964,
+      "eval_runtime": 38.0362,
+      "eval_samples_per_second": 52.582,
+      "eval_steps_per_second": 1.656,
+      "step": 100
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 54.47861812243419,
+      "learning_rate": 7.569444444444444e-07,
+      "logits/chosen": -3.544140577316284,
+      "logits/rejected": -3.6400389671325684,
+      "logps/chosen": -299.38751220703125,
+      "logps/rejected": -278.86248779296875,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.4984374940395355,
+      "rewards/chosen": 0.52374267578125,
+      "rewards/margins": 0.2876220643520355,
+      "rewards/rejected": 0.23598632216453552,
+      "step": 110
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 54.2529317388057,
+      "learning_rate": 8.263888888888888e-07,
+      "logits/chosen": -3.548632860183716,
+      "logits/rejected": -3.6431641578674316,
+      "logps/chosen": -312.3374938964844,
+      "logps/rejected": -277.4125061035156,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": 0.6037353277206421,
+      "rewards/margins": 0.32655030488967896,
+      "rewards/rejected": 0.27708739042282104,
+      "step": 120
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 51.707378579293625,
+      "learning_rate": 8.958333333333334e-07,
+      "logits/chosen": -3.506640672683716,
+      "logits/rejected": -3.6087889671325684,
+      "logps/chosen": -297.2749938964844,
+      "logps/rejected": -261.42498779296875,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.5296875238418579,
+      "rewards/chosen": 0.629077136516571,
+      "rewards/margins": 0.37315672636032104,
+      "rewards/rejected": 0.2562011778354645,
+      "step": 130
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 51.76675306699964,
+      "learning_rate": 9.652777777777777e-07,
+      "logits/chosen": -3.630078077316284,
+      "logits/rejected": -3.742382764816284,
+      "logps/chosen": -299.3374938964844,
+      "logps/rejected": -271.5249938964844,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.5453125238418579,
+      "rewards/chosen": 0.705810546875,
+      "rewards/margins": 0.40642088651657104,
+      "rewards/rejected": 0.2996582090854645,
+      "step": 140
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 55.45929707393807,
+      "learning_rate": 9.999629323566322e-07,
+      "logits/chosen": -3.5044922828674316,
+      "logits/rejected": -3.6474609375,
+      "logps/chosen": -306.5625,
+      "logps/rejected": -272.1937561035156,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": 0.821118175983429,
+      "rewards/margins": 0.4505615234375,
+      "rewards/rejected": 0.37067872285842896,
+      "step": 150
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 55.94064778962465,
+      "learning_rate": 9.996664241851196e-07,
+      "logits/chosen": -3.5894532203674316,
+      "logits/rejected": -3.652539014816284,
+      "logps/chosen": -292.4750061035156,
+      "logps/rejected": -262.2875061035156,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": 0.7660888433456421,
+      "rewards/margins": 0.4485107362270355,
+      "rewards/rejected": 0.3175415098667145,
+      "step": 160
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 48.76264393342852,
+      "learning_rate": 9.990735836893225e-07,
+      "logits/chosen": -3.4935545921325684,
+      "logits/rejected": -3.5982422828674316,
+      "logps/chosen": -302.7250061035156,
+      "logps/rejected": -280.07501220703125,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.5804687738418579,
+      "rewards/chosen": 0.786694347858429,
+      "rewards/margins": 0.560546875,
+      "rewards/rejected": 0.22648926079273224,
+      "step": 170
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 54.177189957274386,
+      "learning_rate": 9.98184762459409e-07,
+      "logits/chosen": -3.520312547683716,
+      "logits/rejected": -3.6558594703674316,
+      "logps/chosen": -296.61248779296875,
+      "logps/rejected": -268.2124938964844,
+      "loss": 0.6264,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.740039050579071,
+      "rewards/margins": 0.5406860113143921,
+      "rewards/rejected": 0.19941405951976776,
+      "step": 180
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 49.8040051805951,
+      "learning_rate": 9.97000487619973e-07,
+      "logits/chosen": -3.4957032203674316,
+      "logits/rejected": -3.6705079078674316,
+      "logps/chosen": -300.3374938964844,
+      "logps/rejected": -266.92498779296875,
+      "loss": 0.6251,
+      "rewards/accuracies": 0.6070312261581421,
+      "rewards/chosen": 0.781787097454071,
+      "rewards/margins": 0.5819457769393921,
+      "rewards/rejected": 0.20030517876148224,
+      "step": 190
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 48.04696461347598,
+      "learning_rate": 9.955214615174171e-07,
+      "logits/chosen": -3.5367188453674316,
+      "logits/rejected": -3.603320360183716,
+      "logps/chosen": -308.375,
+      "logps/rejected": -281.48748779296875,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.5835937261581421,
+      "rewards/chosen": 0.722363293170929,
+      "rewards/margins": 0.549731433391571,
+      "rewards/rejected": 0.17241211235523224,
+      "step": 200
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "eval_logits/chosen": -3.527033805847168,
+      "eval_logits/rejected": -3.595982074737549,
+      "eval_logps/chosen": -296.4603271484375,
+      "eval_logps/rejected": -278.0952453613281,
+      "eval_loss": 0.6377070546150208,
+      "eval_rewards/accuracies": 0.6016865372657776,
+      "eval_rewards/chosen": 0.6889880895614624,
+      "eval_rewards/margins": 0.5590742826461792,
+      "eval_rewards/rejected": 0.1302083283662796,
+      "eval_runtime": 38.0584,
+      "eval_samples_per_second": 52.551,
+      "eval_steps_per_second": 1.655,
+      "step": 200
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 80.48182934058285,
+      "learning_rate": 9.937485613034207e-07,
+      "logits/chosen": -3.5794920921325684,
+      "logits/rejected": -3.6787109375,
+      "logps/chosen": -303.86248779296875,
+      "logps/rejected": -259.1625061035156,
+      "loss": 0.6132,
+      "rewards/accuracies": 0.594531238079071,
+      "rewards/chosen": 0.788500964641571,
+      "rewards/margins": 0.6539551019668579,
+      "rewards/rejected": 0.13487549126148224,
+      "step": 210
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 47.44272816022087,
+      "learning_rate": 9.91682838414733e-07,
+      "logits/chosen": -3.5999999046325684,
+      "logits/rejected": -3.680468797683716,
+      "logps/chosen": -301.1625061035156,
+      "logps/rejected": -267.0375061035156,
+      "loss": 0.6408,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.7625366449356079,
+      "rewards/margins": 0.5675048828125,
+      "rewards/rejected": 0.19476318359375,
+      "step": 220
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 46.318603241911035,
+      "learning_rate": 9.893255179496105e-07,
+      "logits/chosen": -3.5074219703674316,
+      "logits/rejected": -3.6119141578674316,
+      "logps/chosen": -312.2875061035156,
+      "logps/rejected": -281.8374938964844,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.5882812738418579,
+      "rewards/chosen": 0.774340808391571,
+      "rewards/margins": 0.621826171875,
+      "rewards/rejected": 0.15242919325828552,
+      "step": 230
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 48.09344714285963,
+      "learning_rate": 9.866779979412583e-07,
+      "logits/chosen": -3.5160155296325684,
+      "logits/rejected": -3.618457078933716,
+      "logps/chosen": -304.42498779296875,
+      "logps/rejected": -262.875,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": 0.690051257610321,
+      "rewards/margins": 0.5989013910293579,
+      "rewards/rejected": 0.09129638969898224,
+      "step": 240
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 61.23401609930622,
+      "learning_rate": 9.837418485287126e-07,
+      "logits/chosen": -3.5601563453674316,
+      "logits/rejected": -3.639453172683716,
+      "logps/chosen": -322.13751220703125,
+      "logps/rejected": -293.6499938964844,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.5796874761581421,
+      "rewards/chosen": 0.6351318359375,
+      "rewards/margins": 0.581835925579071,
+      "rewards/rejected": 0.05330810695886612,
+      "step": 250
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 68.04230603151362,
+      "learning_rate": 9.805188110256532e-07,
+      "logits/chosen": -3.5179686546325684,
+      "logits/rejected": -3.6324219703674316,
+      "logps/chosen": -288.54998779296875,
+      "logps/rejected": -270.67498779296875,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.641894519329071,
+      "rewards/margins": 0.591137707233429,
+      "rewards/rejected": 0.05062256008386612,
+      "step": 260
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 50.34033381519043,
+      "learning_rate": 9.770107968877002e-07,
+      "logits/chosen": -3.50390625,
+      "logits/rejected": -3.5873045921325684,
+      "logps/chosen": -297.2749938964844,
+      "logps/rejected": -267.7124938964844,
+      "loss": 0.624,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.7995849847793579,
+      "rewards/margins": 0.6589111089706421,
+      "rewards/rejected": 0.140625,
+      "step": 270
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 47.154652871656815,
+      "learning_rate": 9.732198865788047e-07,
+      "logits/chosen": -3.509765625,
+      "logits/rejected": -3.638476610183716,
+      "logps/chosen": -295.11248779296875,
+      "logps/rejected": -269.125,
+      "loss": 0.6247,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": 0.791064441204071,
+      "rewards/margins": 0.646130383014679,
+      "rewards/rejected": 0.14484862983226776,
+      "step": 280
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 44.955804750639444,
+      "learning_rate": 9.691483283374083e-07,
+      "logits/chosen": -3.4517579078674316,
+      "logits/rejected": -3.557812452316284,
+      "logps/chosen": -299.45001220703125,
+      "logps/rejected": -271.1000061035156,
+      "loss": 0.6223,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.8304198980331421,
+      "rewards/margins": 0.637768566608429,
+      "rewards/rejected": 0.19233398139476776,
+      "step": 290
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 54.37341284929218,
+      "learning_rate": 9.647985368431032e-07,
+      "logits/chosen": -3.509570360183716,
+      "logits/rejected": -3.634960889816284,
+      "logps/chosen": -307.0375061035156,
+      "logps/rejected": -288.2749938964844,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.8645263910293579,
+      "rewards/margins": 0.7024902105331421,
+      "rewards/rejected": 0.16193847358226776,
+      "step": 300
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "eval_logits/chosen": -3.507936477661133,
+      "eval_logits/rejected": -3.5768849849700928,
+      "eval_logps/chosen": -296.4920654296875,
+      "eval_logps/rejected": -278.3174743652344,
+      "eval_loss": 0.620190441608429,
+      "eval_rewards/accuracies": 0.6135912537574768,
+      "eval_rewards/chosen": 0.71450275182724,
+      "eval_rewards/margins": 0.6541263461112976,
+      "eval_rewards/rejected": 0.0606708824634552,
+      "eval_runtime": 38.0336,
+      "eval_samples_per_second": 52.585,
+      "eval_steps_per_second": 1.656,
+      "step": 300
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 48.743067345115136,
+      "learning_rate": 9.601730917845796e-07,
+      "logits/chosen": -3.55859375,
+      "logits/rejected": -3.6328125,
+      "logps/chosen": -296.07501220703125,
+      "logps/rejected": -271.70001220703125,
+      "loss": 0.643,
+      "rewards/accuracies": 0.5882812738418579,
+      "rewards/chosen": 0.7311767339706421,
+      "rewards/margins": 0.5884033441543579,
+      "rewards/rejected": 0.14277343451976776,
+      "step": 310
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 48.07476536452811,
+      "learning_rate": 9.552747363297171e-07,
+      "logits/chosen": -3.4652342796325684,
+      "logits/rejected": -3.585156202316284,
+      "logps/chosen": -298.8500061035156,
+      "logps/rejected": -260.61248779296875,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.6460937261581421,
+      "rewards/chosen": 0.889575183391571,
+      "rewards/margins": 0.790771484375,
+      "rewards/rejected": 0.09854736179113388,
+      "step": 320
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 53.46841975260531,
+      "learning_rate": 9.501063754987187e-07,
+      "logits/chosen": -3.550585985183716,
+      "logits/rejected": -3.6207032203674316,
+      "logps/chosen": -296.98748779296875,
+      "logps/rejected": -262.95001220703125,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.6226562261581421,
+      "rewards/chosen": 0.919677734375,
+      "rewards/margins": 0.746594250202179,
+      "rewards/rejected": 0.17322997748851776,
+      "step": 330
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 46.52116347901424,
+      "learning_rate": 9.446710744412594e-07,
+      "logits/chosen": -3.515625,
+      "logits/rejected": -3.603710889816284,
+      "logps/chosen": -304.57501220703125,
+      "logps/rejected": -271.7749938964844,
+      "loss": 0.6273,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": 0.9882568120956421,
+      "rewards/margins": 0.75140380859375,
+      "rewards/rejected": 0.23686523735523224,
+      "step": 340
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 50.500544854935896,
+      "learning_rate": 9.38972056618668e-07,
+      "logits/chosen": -3.452831983566284,
+      "logits/rejected": -3.5414061546325684,
+      "logps/chosen": -311.75,
+      "logps/rejected": -276.125,
+      "loss": 0.627,
+      "rewards/accuracies": 0.6070312261581421,
+      "rewards/chosen": 0.831591784954071,
+      "rewards/margins": 0.6449950933456421,
+      "rewards/rejected": 0.18647460639476776,
+      "step": 350
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 46.93234221408032,
+      "learning_rate": 9.330127018922193e-07,
+      "logits/chosen": -3.4820313453674316,
+      "logits/rejected": -3.5951170921325684,
+      "logps/chosen": -314.88751220703125,
+      "logps/rejected": -279.5625,
+      "loss": 0.6304,
+      "rewards/accuracies": 0.610156238079071,
+      "rewards/chosen": 0.7821899652481079,
+      "rewards/margins": 0.7119995355606079,
+      "rewards/rejected": 0.07012939453125,
+      "step": 360
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 44.9415263678177,
+      "learning_rate": 9.267965445186731e-07,
+      "logits/chosen": -3.462695360183716,
+      "logits/rejected": -3.553515672683716,
+      "logps/chosen": -306.1000061035156,
+      "logps/rejected": -274.4624938964844,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.6273437738418579,
+      "rewards/chosen": 0.8614746332168579,
+      "rewards/margins": 0.8078857660293579,
+      "rewards/rejected": 0.05366211012005806,
+      "step": 370
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 52.52993532420595,
+      "learning_rate": 9.203272710542469e-07,
+      "logits/chosen": -3.4583983421325684,
+      "logits/rejected": -3.5511717796325684,
+      "logps/chosen": -312.32501220703125,
+      "logps/rejected": -274.3500061035156,
+      "loss": 0.6433,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": 0.833251953125,
+      "rewards/margins": 0.7073730230331421,
+      "rewards/rejected": 0.12589111924171448,
+      "step": 380
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 52.202742881471636,
+      "learning_rate": 9.136087181682649e-07,
+      "logits/chosen": -3.557812452316284,
+      "logits/rejected": -3.679882764816284,
+      "logps/chosen": -303.98748779296875,
+      "logps/rejected": -267.79998779296875,
+      "loss": 0.606,
+      "rewards/accuracies": 0.620312511920929,
+      "rewards/chosen": 0.864428699016571,
+      "rewards/margins": 0.752490222454071,
+      "rewards/rejected": 0.11269531399011612,
+      "step": 390
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 47.69618067868101,
+      "learning_rate": 9.066448703677827e-07,
+      "logits/chosen": -3.5224609375,
+      "logits/rejected": -3.626953125,
+      "logps/chosen": -297.98748779296875,
+      "logps/rejected": -278.0625,
+      "loss": 0.6007,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": 0.845019519329071,
+      "rewards/margins": 0.7862793207168579,
+      "rewards/rejected": 0.058837890625,
+      "step": 400
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "eval_logits/chosen": -3.514880895614624,
+      "eval_logits/rejected": -3.5835814476013184,
+      "eval_logps/chosen": -296.3809509277344,
+      "eval_logps/rejected": -278.3174743652344,
+      "eval_loss": 0.6226006150245667,
+      "eval_rewards/accuracies": 0.6259920597076416,
+      "eval_rewards/chosen": 0.7742125391960144,
+      "eval_rewards/margins": 0.7286396622657776,
+      "eval_rewards/rejected": 0.0455109141767025,
+      "eval_runtime": 38.0291,
+      "eval_samples_per_second": 52.591,
+      "eval_steps_per_second": 1.657,
+      "step": 400
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 60.134246518186316,
+      "learning_rate": 8.994398576345334e-07,
+      "logits/chosen": -3.575976610183716,
+      "logits/rejected": -3.637890577316284,
+      "logps/chosen": -310.95001220703125,
+      "logps/rejected": -277.70001220703125,
+      "loss": 0.6046,
+      "rewards/accuracies": 0.6304687261581421,
+      "rewards/chosen": 0.8739989995956421,
+      "rewards/margins": 0.8191894292831421,
+      "rewards/rejected": 0.054443359375,
+      "step": 410
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 50.986473985527944,
+      "learning_rate": 8.919979529756007e-07,
+      "logits/chosen": -3.5517578125,
+      "logits/rejected": -3.6263670921325684,
+      "logps/chosen": -312.63751220703125,
+      "logps/rejected": -279.23748779296875,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.92138671875,
+      "rewards/margins": 0.7955077886581421,
+      "rewards/rejected": 0.12602539360523224,
+      "step": 420
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 51.35655926261148,
+      "learning_rate": 8.843235698892659e-07,
+      "logits/chosen": -3.5326170921325684,
+      "logits/rejected": -3.6015625,
+      "logps/chosen": -309.63751220703125,
+      "logps/rejected": -275.6625061035156,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.6304687261581421,
+      "rewards/chosen": 0.8770507574081421,
+      "rewards/margins": 0.765795886516571,
+      "rewards/rejected": 0.11130370944738388,
+      "step": 430
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 47.21403549154651,
+      "learning_rate": 8.764212597475396e-07,
+      "logits/chosen": -3.4990234375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -322.1875,
+      "logps/rejected": -286.54376220703125,
+      "loss": 0.634,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": 0.8940674066543579,
+      "rewards/margins": 0.78759765625,
+      "rewards/rejected": 0.10640869289636612,
+      "step": 440
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 56.39157595790645,
+      "learning_rate": 8.682957090969219e-07,
+      "logits/chosen": -3.5394530296325684,
+      "logits/rejected": -3.6044921875,
+      "logps/chosen": -295.98748779296875,
+      "logps/rejected": -276.8374938964844,
+      "loss": 0.6167,
+      "rewards/accuracies": 0.624218761920929,
+      "rewards/chosen": 0.8763427734375,
+      "rewards/margins": 0.784863293170929,
+      "rewards/rejected": 0.09123535454273224,
+      "step": 450
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 49.466561222887684,
+      "learning_rate": 8.599517368789979e-07,
+      "logits/chosen": -3.5093750953674316,
+      "logits/rejected": -3.538281202316284,
+      "logps/chosen": -303.9375,
+      "logps/rejected": -280.2749938964844,
+      "loss": 0.6105,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.8824462890625,
+      "rewards/margins": 0.7829955816268921,
+      "rewards/rejected": 0.09906005859375,
+      "step": 460
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 47.5097419947445,
+      "learning_rate": 8.513942915725159e-07,
+      "logits/chosen": -3.4994139671325684,
+      "logits/rejected": -3.54296875,
+      "logps/chosen": -295.6000061035156,
+      "logps/rejected": -281.5375061035156,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.64453125,
+      "rewards/chosen": 0.884716808795929,
+      "rewards/margins": 0.768017590045929,
+      "rewards/rejected": 0.11665038764476776,
+      "step": 470
+    },
+    {
+      "epoch": 1.0041873855011778,
+      "grad_norm": 18.070091348537584,
+      "learning_rate": 8.426284482586396e-07,
+      "logits/chosen": -3.4588067531585693,
+      "logits/rejected": -3.612215995788574,
+      "logps/chosen": -305.98699951171875,
+      "logps/rejected": -285.4545593261719,
+      "loss": 0.5237,
+      "rewards/accuracies": 0.6784632205963135,
+      "rewards/chosen": 1.0996854305267334,
+      "rewards/margins": 1.1744115352630615,
+      "rewards/rejected": -0.07487190514802933,
+      "step": 480
+    },
+    {
+      "epoch": 1.0251243130070662,
+      "grad_norm": 15.600818629027911,
+      "learning_rate": 8.336594056111197e-07,
+      "logits/chosen": -3.573437452316284,
+      "logits/rejected": -3.6224608421325684,
+      "logps/chosen": -301.2250061035156,
+      "logps/rejected": -273.92498779296875,
+      "loss": 0.2342,
+      "rewards/accuracies": 0.9117187261581421,
+      "rewards/chosen": 1.764257788658142,
+      "rewards/margins": 2.560351610183716,
+      "rewards/rejected": -0.795483410358429,
+      "step": 490
+    },
+    {
+      "epoch": 1.0460612405129548,
+      "grad_norm": 19.053546410520852,
+      "learning_rate": 8.244924828131667e-07,
+      "logits/chosen": -3.4867186546325684,
+      "logits/rejected": -3.578320264816284,
+      "logps/chosen": -306.3374938964844,
+      "logps/rejected": -288.79998779296875,
+      "loss": 0.2131,
+      "rewards/accuracies": 0.913281261920929,
+      "rewards/chosen": 1.927148461341858,
+      "rewards/margins": 2.767285108566284,
+      "rewards/rejected": -0.840136706829071,
+      "step": 500
+    },
+    {
+      "epoch": 1.0460612405129548,
+      "eval_logits/chosen": -3.502976179122925,
+      "eval_logits/rejected": -3.570932626724243,
+      "eval_logps/chosen": -296.1111145019531,
+      "eval_logps/rejected": -278.19049072265625,
+      "eval_loss": 0.6569033265113831,
+      "eval_rewards/accuracies": 0.6398809552192688,
+      "eval_rewards/chosen": 0.8705357313156128,
+      "eval_rewards/margins": 0.7919456958770752,
+      "eval_rewards/rejected": 0.0788535475730896,
+      "eval_runtime": 38.0992,
+      "eval_samples_per_second": 52.494,
+      "eval_steps_per_second": 1.654,
+      "step": 500
+    },
+    {
+      "epoch": 1.0669981680188432,
+      "grad_norm": 19.110449142877567,
+      "learning_rate": 8.151331164028543e-07,
+      "logits/chosen": -3.4869141578674316,
+      "logits/rejected": -3.581249952316284,
+      "logps/chosen": -303.92498779296875,
+      "logps/rejected": -278.84375,
+      "loss": 0.2105,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.9580078125,
+      "rewards/margins": 2.773730516433716,
+      "rewards/rejected": -0.816210925579071,
+      "step": 510
+    },
+    {
+      "epoch": 1.0879350955247318,
+      "grad_norm": 19.803904438143206,
+      "learning_rate": 8.055868570489246e-07,
+      "logits/chosen": -3.468945264816284,
+      "logits/rejected": -3.558789014816284,
+      "logps/chosen": -302.46875,
+      "logps/rejected": -285.26251220703125,
+      "loss": 0.2242,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": 1.9019043445587158,
+      "rewards/margins": 2.7503905296325684,
+      "rewards/rejected": -0.847705066204071,
+      "step": 520
+    },
+    {
+      "epoch": 1.1088720230306202,
+      "grad_norm": 20.530766192673937,
+      "learning_rate": 7.958593662589068e-07,
+      "logits/chosen": -3.4931640625,
+      "logits/rejected": -3.584765672683716,
+      "logps/chosen": -286.1000061035156,
+      "logps/rejected": -272.82501220703125,
+      "loss": 0.2213,
+      "rewards/accuracies": 0.913281261920929,
+      "rewards/chosen": 1.89111328125,
+      "rewards/margins": 2.701367139816284,
+      "rewards/rejected": -0.810302734375,
+      "step": 530
+    },
+    {
+      "epoch": 1.1298089505365088,
+      "grad_norm": 16.00951282362298,
+      "learning_rate": 7.859564130215016e-07,
+      "logits/chosen": -3.4677734375,
+      "logits/rejected": -3.556640625,
+      "logps/chosen": -302.53125,
+      "logps/rejected": -264.0625,
+      "loss": 0.2257,
+      "rewards/accuracies": 0.9117187261581421,
+      "rewards/chosen": 1.90625,
+      "rewards/margins": 2.775585889816284,
+      "rewards/rejected": -0.869091808795929,
+      "step": 540
+    },
+    {
+      "epoch": 1.1507458780423974,
+      "grad_norm": 17.43713739928294,
+      "learning_rate": 7.75883870385223e-07,
+      "logits/chosen": -3.5269532203674316,
+      "logits/rejected": -3.6167969703674316,
+      "logps/chosen": -300.625,
+      "logps/rejected": -280.2749938964844,
+      "loss": 0.21,
+      "rewards/accuracies": 0.914843738079071,
+      "rewards/chosen": 1.8914062976837158,
+      "rewards/margins": 2.8544921875,
+      "rewards/rejected": -0.963671863079071,
+      "step": 550
+    },
+    {
+      "epoch": 1.1716828055482857,
+      "grad_norm": 16.166642847253335,
+      "learning_rate": 7.656477119753267e-07,
+      "logits/chosen": -3.494921922683716,
+      "logits/rejected": -3.6490235328674316,
+      "logps/chosen": -312.6499938964844,
+      "logps/rejected": -265.07501220703125,
+      "loss": 0.2192,
+      "rewards/accuracies": 0.9046875238418579,
+      "rewards/chosen": 2.0152344703674316,
+      "rewards/margins": 2.8544921875,
+      "rewards/rejected": -0.8382812738418579,
+      "step": 560
+    },
+    {
+      "epoch": 1.1926197330541743,
+      "grad_norm": 15.875073427913936,
+      "learning_rate": 7.552540084510894e-07,
+      "logits/chosen": -3.462109327316284,
+      "logits/rejected": -3.5980467796325684,
+      "logps/chosen": -302.7250061035156,
+      "logps/rejected": -277.11248779296875,
+      "loss": 0.2112,
+      "rewards/accuracies": 0.9195312261581421,
+      "rewards/chosen": 2.060546875,
+      "rewards/margins": 2.928515672683716,
+      "rewards/rejected": -0.86859130859375,
+      "step": 570
+    },
+    {
+      "epoch": 1.2135566605600627,
+      "grad_norm": 39.46969078499659,
+      "learning_rate": 7.447089239055427e-07,
+      "logits/chosen": -3.3896484375,
+      "logits/rejected": -3.5423827171325684,
+      "logps/chosen": -302.76251220703125,
+      "logps/rejected": -280.48748779296875,
+      "loss": 0.2272,
+      "rewards/accuracies": 0.910937488079071,
+      "rewards/chosen": 2.060742139816284,
+      "rewards/margins": 2.955078125,
+      "rewards/rejected": -0.895214855670929,
+      "step": 580
+    },
+    {
+      "epoch": 1.2344935880659513,
+      "grad_norm": 18.983440764405287,
+      "learning_rate": 7.340187122097929e-07,
+      "logits/chosen": -3.4320311546325684,
+      "logits/rejected": -3.5703125,
+      "logps/chosen": -309.1499938964844,
+      "logps/rejected": -278.61248779296875,
+      "loss": 0.2094,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 1.963769555091858,
+      "rewards/margins": 2.98828125,
+      "rewards/rejected": -1.02435302734375,
+      "step": 590
+    },
+    {
+      "epoch": 1.25543051557184,
+      "grad_norm": 16.504811889013304,
+      "learning_rate": 7.231897133040996e-07,
+      "logits/chosen": -3.460253953933716,
+      "logits/rejected": -3.55859375,
+      "logps/chosen": -290.70001220703125,
+      "logps/rejected": -275.5,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.001660108566284,
+      "rewards/margins": 2.9073243141174316,
+      "rewards/rejected": -0.9055420160293579,
+      "step": 600
+    },
+    {
+      "epoch": 1.25543051557184,
+      "eval_logits/chosen": -3.4898314476013184,
+      "eval_logits/rejected": -3.5572917461395264,
+      "eval_logps/chosen": -295.9047546386719,
+      "eval_logps/rejected": -278.3809509277344,
+      "eval_loss": 0.6247028708457947,
+      "eval_rewards/accuracies": 0.6428571343421936,
+      "eval_rewards/chosen": 0.995194673538208,
+      "eval_rewards/margins": 0.947637677192688,
+      "eval_rewards/rejected": 0.0472780242562294,
+      "eval_runtime": 38.115,
+      "eval_samples_per_second": 52.473,
+      "eval_steps_per_second": 1.653,
+      "step": 600
+    },
+    {
+      "epoch": 1.2763674430777283,
+      "grad_norm": 22.48813190436517,
+      "learning_rate": 7.122283494379075e-07,
+      "logits/chosen": -3.44921875,
+      "logits/rejected": -3.541210889816284,
+      "logps/chosen": -301.54998779296875,
+      "logps/rejected": -272.6000061035156,
+      "loss": 0.2129,
+      "rewards/accuracies": 0.9039062261581421,
+      "rewards/chosen": 2.0810546875,
+      "rewards/margins": 2.9857420921325684,
+      "rewards/rejected": -0.9039062261581421,
+      "step": 610
+    },
+    {
+      "epoch": 1.297304370583617,
+      "grad_norm": 19.402784871843174,
+      "learning_rate": 7.011411213610661e-07,
+      "logits/chosen": -3.5355467796325684,
+      "logits/rejected": -3.5873045921325684,
+      "logps/chosen": -301.6625061035156,
+      "logps/rejected": -271.4750061035156,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.907031238079071,
+      "rewards/chosen": 2.077831983566284,
+      "rewards/margins": 2.98583984375,
+      "rewards/rejected": -0.908374011516571,
+      "step": 620
+    },
+    {
+      "epoch": 1.3182412980895053,
+      "grad_norm": 16.547117779575384,
+      "learning_rate": 6.899346044684928e-07,
+      "logits/chosen": -3.394335985183716,
+      "logits/rejected": -3.565624952316284,
+      "logps/chosen": -310.2124938964844,
+      "logps/rejected": -275.4624938964844,
+      "loss": 0.2067,
+      "rewards/accuracies": 0.899218738079071,
+      "rewards/chosen": 2.2144532203674316,
+      "rewards/margins": 3.063183546066284,
+      "rewards/rejected": -0.848461925983429,
+      "step": 630
+    },
+    {
+      "epoch": 1.339178225595394,
+      "grad_norm": 15.105169958190366,
+      "learning_rate": 6.786154449005663e-07,
+      "logits/chosen": -3.4849610328674316,
+      "logits/rejected": -3.6058592796325684,
+      "logps/chosen": -303.1499938964844,
+      "logps/rejected": -261.32501220703125,
+      "loss": 0.1987,
+      "rewards/accuracies": 0.9164062738418579,
+      "rewards/chosen": 2.111523389816284,
+      "rewards/margins": 3.013378858566284,
+      "rewards/rejected": -0.9011474847793579,
+      "step": 640
+    },
+    {
+      "epoch": 1.3601151531012823,
+      "grad_norm": 21.223016171216557,
+      "learning_rate": 6.671903556015663e-07,
+      "logits/chosen": -3.466796875,
+      "logits/rejected": -3.5234375,
+      "logps/chosen": -292.92498779296875,
+      "logps/rejected": -271.9125061035156,
+      "loss": 0.218,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 1.9792969226837158,
+      "rewards/margins": 3.0326170921325684,
+      "rewards/rejected": -1.053491234779358,
+      "step": 650
+    },
+    {
+      "epoch": 1.3810520806071709,
+      "grad_norm": 14.724238161671572,
+      "learning_rate": 6.556661123384908e-07,
+      "logits/chosen": -3.490429639816284,
+      "logits/rejected": -3.6117186546325684,
+      "logps/chosen": -307.98748779296875,
+      "logps/rejected": -274.6000061035156,
+      "loss": 0.2142,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.9401366710662842,
+      "rewards/margins": 2.953906297683716,
+      "rewards/rejected": -1.0133788585662842,
+      "step": 660
+    },
+    {
+      "epoch": 1.4019890081130595,
+      "grad_norm": 17.903068190572835,
+      "learning_rate": 6.440495496826188e-07,
+      "logits/chosen": -3.5107421875,
+      "logits/rejected": -3.638867139816284,
+      "logps/chosen": -293.4624938964844,
+      "logps/rejected": -270.38751220703125,
+      "loss": 0.225,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 1.975195288658142,
+      "rewards/margins": 2.8541016578674316,
+      "rewards/rejected": -0.879223644733429,
+      "step": 670
+    },
+    {
+      "epoch": 1.4229259356189479,
+      "grad_norm": 16.068043401370257,
+      "learning_rate": 6.323475569561966e-07,
+      "logits/chosen": -3.559765577316284,
+      "logits/rejected": -3.6019530296325684,
+      "logps/chosen": -309.88751220703125,
+      "logps/rejected": -286.2875061035156,
+      "loss": 0.2182,
+      "rewards/accuracies": 0.9039062261581421,
+      "rewards/chosen": 2.1170897483825684,
+      "rewards/margins": 3.052539110183716,
+      "rewards/rejected": -0.935351550579071,
+      "step": 680
+    },
+    {
+      "epoch": 1.4438628631248365,
+      "grad_norm": 22.84533468362057,
+      "learning_rate": 6.205670741466554e-07,
+      "logits/chosen": -3.4888672828674316,
+      "logits/rejected": -3.6283202171325684,
+      "logps/chosen": -296.8374938964844,
+      "logps/rejected": -271.1625061035156,
+      "loss": 0.2183,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.026074171066284,
+      "rewards/margins": 2.9505858421325684,
+      "rewards/rejected": -0.9241943359375,
+      "step": 690
+    },
+    {
+      "epoch": 1.464799790630725,
+      "grad_norm": 18.089193474323352,
+      "learning_rate": 6.087150877907785e-07,
+      "logits/chosen": -3.491992235183716,
+      "logits/rejected": -3.5814452171325684,
+      "logps/chosen": -302.5874938964844,
+      "logps/rejected": -284.04376220703125,
+      "loss": 0.2133,
+      "rewards/accuracies": 0.91015625,
+      "rewards/chosen": 2.099316358566284,
+      "rewards/margins": 3.2081055641174316,
+      "rewards/rejected": -1.1083495616912842,
+      "step": 700
+    },
+    {
+      "epoch": 1.464799790630725,
+      "eval_logits/chosen": -3.4811508655548096,
+      "eval_logits/rejected": -3.549851179122925,
+      "eval_logps/chosen": -295.9206237792969,
+      "eval_logps/rejected": -278.4285583496094,
+      "eval_loss": 0.6190805435180664,
+      "eval_rewards/accuracies": 0.6448412537574768,
+      "eval_rewards/chosen": 0.957821786403656,
+      "eval_rewards/margins": 0.975771963596344,
+      "eval_rewards/rejected": -0.01775638572871685,
+      "eval_runtime": 37.9967,
+      "eval_samples_per_second": 52.636,
+      "eval_steps_per_second": 1.658,
+      "step": 700
+    },
+    {
+      "epoch": 1.4857367181366135,
+      "grad_norm": 16.150705795503985,
+      "learning_rate": 5.96798626831265e-07,
+      "logits/chosen": -3.470507860183716,
+      "logits/rejected": -3.614062547683716,
+      "logps/chosen": -303.6625061035156,
+      "logps/rejected": -276.73748779296875,
+      "loss": 0.198,
+      "rewards/accuracies": 0.9140625,
+      "rewards/chosen": 2.0196776390075684,
+      "rewards/margins": 3.1622071266174316,
+      "rewards/rejected": -1.142968773841858,
+      "step": 710
+    },
+    {
+      "epoch": 1.5066736456425018,
+      "grad_norm": 34.06337514922338,
+      "learning_rate": 5.848247584481423e-07,
+      "logits/chosen": -3.506054639816284,
+      "logits/rejected": -3.5615234375,
+      "logps/chosen": -308.8187561035156,
+      "logps/rejected": -275.875,
+      "loss": 0.2066,
+      "rewards/accuracies": 0.91015625,
+      "rewards/chosen": 2.0882811546325684,
+      "rewards/margins": 3.0848631858825684,
+      "rewards/rejected": -0.9969482421875,
+      "step": 720
+    },
+    {
+      "epoch": 1.5276105731483904,
+      "grad_norm": 20.22654154696713,
+      "learning_rate": 5.728005838675025e-07,
+      "logits/chosen": -3.44921875,
+      "logits/rejected": -3.556445360183716,
+      "logps/chosen": -285.26251220703125,
+      "logps/rejected": -270.5625,
+      "loss": 0.2229,
+      "rewards/accuracies": 0.9046875238418579,
+      "rewards/chosen": 2.0223631858825684,
+      "rewards/margins": 2.955078125,
+      "rewards/rejected": -0.933276355266571,
+      "step": 730
+    },
+    {
+      "epoch": 1.548547500654279,
+      "grad_norm": 21.59266975059039,
+      "learning_rate": 5.607332341500462e-07,
+      "logits/chosen": -3.532421827316284,
+      "logits/rejected": -3.60546875,
+      "logps/chosen": -314.3500061035156,
+      "logps/rejected": -280.17498779296875,
+      "loss": 0.2316,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": 2.1533203125,
+      "rewards/margins": 3.273144483566284,
+      "rewards/rejected": -1.1192626953125,
+      "step": 740
+    },
+    {
+      "epoch": 1.5694844281601674,
+      "grad_norm": 15.46089645661076,
+      "learning_rate": 5.486298659619345e-07,
+      "logits/chosen": -3.4476561546325684,
+      "logits/rejected": -3.5220704078674316,
+      "logps/chosen": -310.92498779296875,
+      "logps/rejected": -285.26251220703125,
+      "loss": 0.2066,
+      "rewards/accuracies": 0.9039062261581421,
+      "rewards/chosen": 2.1983885765075684,
+      "rewards/margins": 3.2510743141174316,
+      "rewards/rejected": -1.052832007408142,
+      "step": 750
+    },
+    {
+      "epoch": 1.590421355666056,
+      "grad_norm": 15.66337481759921,
+      "learning_rate": 5.364976573304537e-07,
+      "logits/chosen": -3.441210985183716,
+      "logits/rejected": -3.6083984375,
+      "logps/chosen": -303.3125,
+      "logps/rejected": -272.57501220703125,
+      "loss": 0.2053,
+      "rewards/accuracies": 0.9195312261581421,
+      "rewards/chosen": 2.228222608566284,
+      "rewards/margins": 3.1898436546325684,
+      "rewards/rejected": -0.961596667766571,
+      "step": 760
+    },
+    {
+      "epoch": 1.6113582831719446,
+      "grad_norm": 17.996816628434395,
+      "learning_rate": 5.243438033870126e-07,
+      "logits/chosen": -3.519335985183716,
+      "logits/rejected": -3.599414110183716,
+      "logps/chosen": -297.26251220703125,
+      "logps/rejected": -276.48748779296875,
+      "loss": 0.2045,
+      "rewards/accuracies": 0.9117187261581421,
+      "rewards/chosen": 2.1712889671325684,
+      "rewards/margins": 3.0947265625,
+      "rewards/rejected": -0.9230102300643921,
+      "step": 770
+    },
+    {
+      "epoch": 1.632295210677833,
+      "grad_norm": 23.596964639903106,
+      "learning_rate": 5.121755120999948e-07,
+      "logits/chosen": -3.529101610183716,
+      "logits/rejected": -3.6044921875,
+      "logps/chosen": -289.36248779296875,
+      "logps/rejected": -274.26251220703125,
+      "loss": 0.2182,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 2.1529297828674316,
+      "rewards/margins": 3.0741209983825684,
+      "rewards/rejected": -0.922045886516571,
+      "step": 780
+    },
+    {
+      "epoch": 1.6532321381837214,
+      "grad_norm": 21.08077712966235,
+      "learning_rate": 5e-07,
+      "logits/chosen": -3.512890577316284,
+      "logits/rejected": -3.6097655296325684,
+      "logps/chosen": -295.13751220703125,
+      "logps/rejected": -260.3374938964844,
+      "loss": 0.2216,
+      "rewards/accuracies": 0.8984375,
+      "rewards/chosen": 2.055371046066284,
+      "rewards/margins": 2.9833006858825684,
+      "rewards/rejected": -0.928173840045929,
+      "step": 790
+    },
+    {
+      "epoch": 1.6741690656896102,
+      "grad_norm": 15.572269688434925,
+      "learning_rate": 4.878244879000052e-07,
+      "logits/chosen": -3.5003905296325684,
+      "logits/rejected": -3.5376954078674316,
+      "logps/chosen": -291.11248779296875,
+      "logps/rejected": -277.42498779296875,
+      "loss": 0.2187,
+      "rewards/accuracies": 0.905468761920929,
+      "rewards/chosen": 2.0094237327575684,
+      "rewards/margins": 3.084179639816284,
+      "rewards/rejected": -1.074926733970642,
+      "step": 800
+    },
+    {
+      "epoch": 1.6741690656896102,
+      "eval_logits/chosen": -3.486359119415283,
+      "eval_logits/rejected": -3.5562996864318848,
+      "eval_logps/chosen": -296.047607421875,
+      "eval_logps/rejected": -278.5396728515625,
+      "eval_loss": 0.6094306707382202,
+      "eval_rewards/accuracies": 0.6552579402923584,
+      "eval_rewards/chosen": 0.9073195457458496,
+      "eval_rewards/margins": 1.0078125,
+      "eval_rewards/rejected": -0.1005084291100502,
+      "eval_runtime": 38.3614,
+      "eval_samples_per_second": 52.136,
+      "eval_steps_per_second": 1.642,
+      "step": 800
+    },
+    {
+      "epoch": 1.6951059931954986,
+      "grad_norm": 18.291248263940155,
+      "learning_rate": 4.756561966129875e-07,
+      "logits/chosen": -3.504101514816284,
+      "logits/rejected": -3.583789110183716,
+      "logps/chosen": -315.7124938964844,
+      "logps/rejected": -279.11248779296875,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.9078124761581421,
+      "rewards/chosen": 2.085253953933716,
+      "rewards/margins": 3.173828125,
+      "rewards/rejected": -1.087377905845642,
+      "step": 810
+    },
+    {
+      "epoch": 1.716042920701387,
+      "grad_norm": 19.91412175514064,
+      "learning_rate": 4.635023426695462e-07,
+      "logits/chosen": -3.474609375,
+      "logits/rejected": -3.588085889816284,
+      "logps/chosen": -301.7749938964844,
+      "logps/rejected": -270.375,
+      "loss": 0.2,
+      "rewards/accuracies": 0.9085937738418579,
+      "rewards/chosen": 2.0853514671325684,
+      "rewards/margins": 3.1807618141174316,
+      "rewards/rejected": -1.0952880382537842,
+      "step": 820
+    },
+    {
+      "epoch": 1.7369798482072756,
+      "grad_norm": 22.170504398138394,
+      "learning_rate": 4.513701340380654e-07,
+      "logits/chosen": -3.550585985183716,
+      "logits/rejected": -3.660351514816284,
+      "logps/chosen": -309.54998779296875,
+      "logps/rejected": -280.8500061035156,
+      "loss": 0.2038,
+      "rewards/accuracies": 0.905468761920929,
+      "rewards/chosen": 2.167187452316284,
+      "rewards/margins": 3.3080077171325684,
+      "rewards/rejected": -1.1409423351287842,
+      "step": 830
+    },
+    {
+      "epoch": 1.7579167757131642,
+      "grad_norm": 22.202916599466086,
+      "learning_rate": 4.392667658499539e-07,
+      "logits/chosen": -3.466601610183716,
+      "logits/rejected": -3.598437547683716,
+      "logps/chosen": -299.5562438964844,
+      "logps/rejected": -272.875,
+      "loss": 0.2183,
+      "rewards/accuracies": 0.8968750238418579,
+      "rewards/chosen": 2.028369188308716,
+      "rewards/margins": 3.08984375,
+      "rewards/rejected": -1.0598633289337158,
+      "step": 840
+    },
+    {
+      "epoch": 1.7788537032190526,
+      "grad_norm": 15.583732129981541,
+      "learning_rate": 4.2719941613249767e-07,
+      "logits/chosen": -3.495898485183716,
+      "logits/rejected": -3.6175780296325684,
+      "logps/chosen": -313.5249938964844,
+      "logps/rejected": -280.2124938964844,
+      "loss": 0.1954,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 2.173632860183716,
+      "rewards/margins": 3.3160157203674316,
+      "rewards/rejected": -1.1440918445587158,
+      "step": 850
+    },
+    {
+      "epoch": 1.7997906307249412,
+      "grad_norm": 20.66642843378369,
+      "learning_rate": 4.1517524155185767e-07,
+      "logits/chosen": -3.495312452316284,
+      "logits/rejected": -3.556640625,
+      "logps/chosen": -282.0,
+      "logps/rejected": -269.95001220703125,
+      "loss": 0.2134,
+      "rewards/accuracies": 0.907031238079071,
+      "rewards/chosen": 1.9568359851837158,
+      "rewards/margins": 3.0595703125,
+      "rewards/rejected": -1.1016356945037842,
+      "step": 860
+    },
+    {
+      "epoch": 1.8207275582308298,
+      "grad_norm": 13.807687571734952,
+      "learning_rate": 4.03201373168735e-07,
+      "logits/chosen": -3.4886717796325684,
+      "logits/rejected": -3.617382764816284,
+      "logps/chosen": -309.46875,
+      "logps/rejected": -274.8500061035156,
+      "loss": 0.1972,
+      "rewards/accuracies": 0.921093761920929,
+      "rewards/chosen": 2.2154297828674316,
+      "rewards/margins": 3.3506836891174316,
+      "rewards/rejected": -1.1370849609375,
+      "step": 870
+    },
+    {
+      "epoch": 1.8416644857367181,
+      "grad_norm": 20.342933608049247,
+      "learning_rate": 3.9128491220922156e-07,
+      "logits/chosen": -3.4341797828674316,
+      "logits/rejected": -3.5394530296325684,
+      "logps/chosen": -297.79998779296875,
+      "logps/rejected": -273.91876220703125,
+      "loss": 0.2024,
+      "rewards/accuracies": 0.914843738079071,
+      "rewards/chosen": 2.114453077316284,
+      "rewards/margins": 3.178027391433716,
+      "rewards/rejected": -1.063989281654358,
+      "step": 880
+    },
+    {
+      "epoch": 1.8626014132426065,
+      "grad_norm": 18.329508428371092,
+      "learning_rate": 3.794329258533446e-07,
+      "logits/chosen": -3.4476561546325684,
+      "logits/rejected": -3.5712890625,
+      "logps/chosen": -294.2749938964844,
+      "logps/rejected": -282.9750061035156,
+      "loss": 0.1985,
+      "rewards/accuracies": 0.9203125238418579,
+      "rewards/chosen": 2.049267530441284,
+      "rewards/margins": 3.168652296066284,
+      "rewards/rejected": -1.1173584461212158,
+      "step": 890
+    },
+    {
+      "epoch": 1.8835383407484951,
+      "grad_norm": 17.507962455507613,
+      "learning_rate": 3.6765244304380315e-07,
+      "logits/chosen": -3.520703077316284,
+      "logits/rejected": -3.631640672683716,
+      "logps/chosen": -306.57501220703125,
+      "logps/rejected": -270.88751220703125,
+      "loss": 0.1981,
+      "rewards/accuracies": 0.917187511920929,
+      "rewards/chosen": 2.2388672828674316,
+      "rewards/margins": 3.3658204078674316,
+      "rewards/rejected": -1.1268310546875,
+      "step": 900
+    },
+    {
+      "epoch": 1.8835383407484951,
+      "eval_logits/chosen": -3.498511791229248,
+      "eval_logits/rejected": -3.566716194152832,
+      "eval_logps/chosen": -295.952392578125,
+      "eval_logps/rejected": -278.4444580078125,
+      "eval_loss": 0.6092287302017212,
+      "eval_rewards/accuracies": 0.6547619104385376,
+      "eval_rewards/chosen": 0.9763144850730896,
+      "eval_rewards/margins": 1.0385664701461792,
+      "eval_rewards/rejected": -0.0626395121216774,
+      "eval_runtime": 38.1205,
+      "eval_samples_per_second": 52.465,
+      "eval_steps_per_second": 1.653,
+      "step": 900
+    },
+    {
+      "epoch": 1.9044752682543837,
+      "grad_norm": 15.055521452243642,
+      "learning_rate": 3.559504503173812e-07,
+      "logits/chosen": -3.533007860183716,
+      "logits/rejected": -3.624218702316284,
+      "logps/chosen": -303.6625061035156,
+      "logps/rejected": -274.7562561035156,
+      "loss": 0.1838,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 2.138867139816284,
+      "rewards/margins": 3.143847703933716,
+      "rewards/rejected": -1.004492163658142,
+      "step": 910
+    },
+    {
+      "epoch": 1.9254121957602721,
+      "grad_norm": 30.77883455970536,
+      "learning_rate": 3.4433388766150915e-07,
+      "logits/chosen": -3.4443359375,
+      "logits/rejected": -3.5840821266174316,
+      "logps/chosen": -297.07501220703125,
+      "logps/rejected": -272.79998779296875,
+      "loss": 0.2049,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 2.1675782203674316,
+      "rewards/margins": 3.213085889816284,
+      "rewards/rejected": -1.045922875404358,
+      "step": 920
+    },
+    {
+      "epoch": 1.9463491232661607,
+      "grad_norm": 26.320172624569338,
+      "learning_rate": 3.328096443984337e-07,
+      "logits/chosen": -3.4111328125,
+      "logits/rejected": -3.494921922683716,
+      "logps/chosen": -303.9125061035156,
+      "logps/rejected": -277.70001220703125,
+      "loss": 0.2094,
+      "rewards/accuracies": 0.901562511920929,
+      "rewards/chosen": 2.2093749046325684,
+      "rewards/margins": 3.2474608421325684,
+      "rewards/rejected": -1.0389404296875,
+      "step": 930
+    },
+    {
+      "epoch": 1.9672860507720493,
+      "grad_norm": 19.16797392404003,
+      "learning_rate": 3.213845550994336e-07,
+      "logits/chosen": -3.5191407203674316,
+      "logits/rejected": -3.5894532203674316,
+      "logps/chosen": -302.6000061035156,
+      "logps/rejected": -292.125,
+      "loss": 0.2105,
+      "rewards/accuracies": 0.9085937738418579,
+      "rewards/chosen": 2.1844725608825684,
+      "rewards/margins": 3.232714891433716,
+      "rewards/rejected": -1.047937035560608,
+      "step": 940
+    },
+    {
+      "epoch": 1.9882229782779377,
+      "grad_norm": 23.999835236297592,
+      "learning_rate": 3.1006539553150724e-07,
+      "logits/chosen": -3.5367188453674316,
+      "logits/rejected": -3.616406202316284,
+      "logps/chosen": -311.07501220703125,
+      "logps/rejected": -275.7749938964844,
+      "loss": 0.2083,
+      "rewards/accuracies": 0.9117187261581421,
+      "rewards/chosen": 2.158886671066284,
+      "rewards/margins": 3.2054686546325684,
+      "rewards/rejected": -1.0462646484375,
+      "step": 950
+    },
+    {
+      "epoch": 2.0083747710023556,
+      "grad_norm": 6.155071574525674,
+      "learning_rate": 2.9885887863893386e-07,
+      "logits/chosen": -3.4608359336853027,
+      "logits/rejected": -3.5858359336853027,
+      "logps/chosen": -308.6883239746094,
+      "logps/rejected": -274.2597351074219,
+      "loss": 0.1594,
+      "rewards/accuracies": 0.9389069676399231,
+      "rewards/chosen": 2.2458908557891846,
+      "rewards/margins": 3.5668628215789795,
+      "rewards/rejected": -1.3198051452636719,
+      "step": 960
+    },
+    {
+      "epoch": 2.029311698508244,
+      "grad_norm": 6.168365854472213,
+      "learning_rate": 2.8777165056209254e-07,
+      "logits/chosen": -3.5560545921325684,
+      "logits/rejected": -3.625781297683716,
+      "logps/chosen": -295.8062438964844,
+      "logps/rejected": -270.5874938964844,
+      "loss": 0.0888,
+      "rewards/accuracies": 0.9820312261581421,
+      "rewards/chosen": 2.466015577316284,
+      "rewards/margins": 4.127734184265137,
+      "rewards/rejected": -1.661865234375,
+      "step": 970
+    },
+    {
+      "epoch": 2.0502486260141324,
+      "grad_norm": 5.083094636735314,
+      "learning_rate": 2.7681028669590033e-07,
+      "logits/chosen": -3.412304639816284,
+      "logits/rejected": -3.532421827316284,
+      "logps/chosen": -314.0625,
+      "logps/rejected": -272.57501220703125,
+      "loss": 0.0854,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 2.6949219703674316,
+      "rewards/margins": 4.290820121765137,
+      "rewards/rejected": -1.5948486328125,
+      "step": 980
+    },
+    {
+      "epoch": 2.0711855535200208,
+      "grad_norm": 6.61063078853505,
+      "learning_rate": 2.659812877902069e-07,
+      "logits/chosen": -3.4888672828674316,
+      "logits/rejected": -3.6005859375,
+      "logps/chosen": -297.61248779296875,
+      "logps/rejected": -279.5874938964844,
+      "loss": 0.0966,
+      "rewards/accuracies": 0.973437488079071,
+      "rewards/chosen": 2.543750047683716,
+      "rewards/margins": 4.118750095367432,
+      "rewards/rejected": -1.5730469226837158,
+      "step": 990
+    },
+    {
+      "epoch": 2.0921224810259096,
+      "grad_norm": 5.943450637086841,
+      "learning_rate": 2.5529107609445735e-07,
+      "logits/chosen": -3.4468750953674316,
+      "logits/rejected": -3.5875000953674316,
+      "logps/chosen": -309.42498779296875,
+      "logps/rejected": -285.3125,
+      "loss": 0.0929,
+      "rewards/accuracies": 0.9703124761581421,
+      "rewards/chosen": 2.6815428733825684,
+      "rewards/margins": 4.2490234375,
+      "rewards/rejected": -1.567529320716858,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0921224810259096,
+      "eval_logits/chosen": -3.4841270446777344,
+      "eval_logits/rejected": -3.55332350730896,
+      "eval_logps/chosen": -295.80950927734375,
+      "eval_logps/rejected": -278.4920654296875,
+      "eval_loss": 0.6299723982810974,
+      "eval_rewards/accuracies": 0.6453372836112976,
+      "eval_rewards/chosen": 1.0352802276611328,
+      "eval_rewards/margins": 1.1254340410232544,
+      "eval_rewards/rejected": -0.0904250368475914,
+      "eval_runtime": 38.1128,
+      "eval_samples_per_second": 52.476,
+      "eval_steps_per_second": 1.653,
+      "step": 1000
+    },
+    {
+      "epoch": 2.113059408531798,
+      "grad_norm": 7.410681037120214,
+      "learning_rate": 2.447459915489106e-07,
+      "logits/chosen": -3.48046875,
+      "logits/rejected": -3.602734327316284,
+      "logps/chosen": -293.54998779296875,
+      "logps/rejected": -270.98748779296875,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.673632860183716,
+      "rewards/margins": 4.233788967132568,
+      "rewards/rejected": -1.558203101158142,
+      "step": 1010
+    },
+    {
+      "epoch": 2.1339963360376863,
+      "grad_norm": 7.934744287733001,
+      "learning_rate": 2.3435228802467337e-07,
+      "logits/chosen": -3.5472655296325684,
+      "logits/rejected": -3.628124952316284,
+      "logps/chosen": -298.5,
+      "logps/rejected": -264.8187561035156,
+      "loss": 0.0898,
+      "rewards/accuracies": 0.9789062738418579,
+      "rewards/chosen": 2.6216797828674316,
+      "rewards/margins": 4.183007717132568,
+      "rewards/rejected": -1.560693383216858,
+      "step": 1020
+    },
+    {
+      "epoch": 2.154933263543575,
+      "grad_norm": 7.6005767777011215,
+      "learning_rate": 2.24116129614777e-07,
+      "logits/chosen": -3.463671922683716,
+      "logits/rejected": -3.5335936546325684,
+      "logps/chosen": -294.11248779296875,
+      "logps/rejected": -276.25,
+      "loss": 0.0841,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.6776366233825684,
+      "rewards/margins": 4.297070503234863,
+      "rewards/rejected": -1.617578148841858,
+      "step": 1030
+    },
+    {
+      "epoch": 2.1758701910494636,
+      "grad_norm": 6.242124155668628,
+      "learning_rate": 2.1404358697849856e-07,
+      "logits/chosen": -3.4683594703674316,
+      "logits/rejected": -3.5501952171325684,
+      "logps/chosen": -311.4750061035156,
+      "logps/rejected": -274.0249938964844,
+      "loss": 0.0938,
+      "rewards/accuracies": 0.9742187261581421,
+      "rewards/chosen": 2.825000047683716,
+      "rewards/margins": 4.315820217132568,
+      "rewards/rejected": -1.489160180091858,
+      "step": 1040
+    },
+    {
+      "epoch": 2.196807118555352,
+      "grad_norm": 6.606473981543361,
+      "learning_rate": 2.0414063374109326e-07,
+      "logits/chosen": -3.5498046875,
+      "logits/rejected": -3.628710985183716,
+      "logps/chosen": -308.9750061035156,
+      "logps/rejected": -274.6625061035156,
+      "loss": 0.0878,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.75439453125,
+      "rewards/margins": 4.4267578125,
+      "rewards/rejected": -1.6730468273162842,
+      "step": 1050
+    },
+    {
+      "epoch": 2.2177440460612403,
+      "grad_norm": 7.208818006351197,
+      "learning_rate": 1.9441314295107535e-07,
+      "logits/chosen": -3.49609375,
+      "logits/rejected": -3.58203125,
+      "logps/chosen": -293.1000061035156,
+      "logps/rejected": -283.4750061035156,
+      "loss": 0.089,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.740429639816284,
+      "rewards/margins": 4.446875095367432,
+      "rewards/rejected": -1.707177758216858,
+      "step": 1060
+    },
+    {
+      "epoch": 2.238680973567129,
+      "grad_norm": 6.134311387575375,
+      "learning_rate": 1.8486688359714564e-07,
+      "logits/chosen": -3.533203125,
+      "logits/rejected": -3.626757860183716,
+      "logps/chosen": -295.2124938964844,
+      "logps/rejected": -273.1875,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.973437488079071,
+      "rewards/chosen": 2.8125977516174316,
+      "rewards/margins": 4.4755859375,
+      "rewards/rejected": -1.66357421875,
+      "step": 1070
+    },
+    {
+      "epoch": 2.2596179010730175,
+      "grad_norm": 5.58334303128779,
+      "learning_rate": 1.7550751718683337e-07,
+      "logits/chosen": -3.4765625,
+      "logits/rejected": -3.59765625,
+      "logps/chosen": -297.29998779296875,
+      "logps/rejected": -287.48748779296875,
+      "loss": 0.0841,
+      "rewards/accuracies": 0.979687511920929,
+      "rewards/chosen": 2.7369141578674316,
+      "rewards/margins": 4.4697265625,
+      "rewards/rejected": -1.7322266101837158,
+      "step": 1080
+    },
+    {
+      "epoch": 2.280554828578906,
+      "grad_norm": 6.5315710194144545,
+      "learning_rate": 1.6634059438888032e-07,
+      "logits/chosen": -3.4320311546325684,
+      "logits/rejected": -3.6244139671325684,
+      "logps/chosen": -303.95001220703125,
+      "logps/rejected": -269.3125,
+      "loss": 0.0868,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.6922850608825684,
+      "rewards/margins": 4.252148628234863,
+      "rewards/rejected": -1.5595703125,
+      "step": 1090
+    },
+    {
+      "epoch": 2.3014917560847947,
+      "grad_norm": 4.9841820667381,
+      "learning_rate": 1.573715517413604e-07,
+      "logits/chosen": -3.5439453125,
+      "logits/rejected": -3.5999999046325684,
+      "logps/chosen": -299.82501220703125,
+      "logps/rejected": -267.9375,
+      "loss": 0.0864,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.6493163108825684,
+      "rewards/margins": 4.379101753234863,
+      "rewards/rejected": -1.7283203601837158,
+      "step": 1100
+    },
+    {
+      "epoch": 2.3014917560847947,
+      "eval_logits/chosen": -3.478670597076416,
+      "eval_logits/rejected": -3.5466270446777344,
+      "eval_logps/chosen": -295.8412780761719,
+      "eval_logps/rejected": -278.69842529296875,
+      "eval_loss": 0.6405720114707947,
+      "eval_rewards/accuracies": 0.653769850730896,
+      "eval_rewards/chosen": 1.0408296585083008,
+      "eval_rewards/margins": 1.1919022798538208,
+      "eval_rewards/rejected": -0.1506773978471756,
+      "eval_runtime": 38.3286,
+      "eval_samples_per_second": 52.18,
+      "eval_steps_per_second": 1.644,
+      "step": 1100
+    },
+    {
+      "epoch": 2.322428683590683,
+      "grad_norm": 6.22314482880758,
+      "learning_rate": 1.486057084274841e-07,
+      "logits/chosen": -3.492968797683716,
+      "logits/rejected": -3.542187452316284,
+      "logps/chosen": -289.07501220703125,
+      "logps/rejected": -277.75,
+      "loss": 0.0882,
+      "rewards/accuracies": 0.979687511920929,
+      "rewards/chosen": 2.667187452316284,
+      "rewards/margins": 4.377539157867432,
+      "rewards/rejected": -1.710107445716858,
+      "step": 1110
+    },
+    {
+      "epoch": 2.3433656110965715,
+      "grad_norm": 8.430802135114272,
+      "learning_rate": 1.4004826312100216e-07,
+      "logits/chosen": -3.449414014816284,
+      "logits/rejected": -3.546093702316284,
+      "logps/chosen": -313.86248779296875,
+      "logps/rejected": -285.57501220703125,
+      "loss": 0.0893,
+      "rewards/accuracies": 0.969531238079071,
+      "rewards/chosen": 2.8282227516174316,
+      "rewards/margins": 4.535742282867432,
+      "rewards/rejected": -1.707666039466858,
+      "step": 1120
+    },
+    {
+      "epoch": 2.36430253860246,
+      "grad_norm": 5.409263427479619,
+      "learning_rate": 1.3170429090307822e-07,
+      "logits/chosen": -3.508984327316284,
+      "logits/rejected": -3.663281202316284,
+      "logps/chosen": -309.1625061035156,
+      "logps/rejected": -272.3500061035156,
+      "loss": 0.0842,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.873339891433716,
+      "rewards/margins": 4.411523342132568,
+      "rewards/rejected": -1.5381348133087158,
+      "step": 1130
+    },
+    {
+      "epoch": 2.3852394661083487,
+      "grad_norm": 6.151010592032672,
+      "learning_rate": 1.235787402524603e-07,
+      "logits/chosen": -3.448437452316284,
+      "logits/rejected": -3.524609327316284,
+      "logps/chosen": -298.7749938964844,
+      "logps/rejected": -277.7749938964844,
+      "loss": 0.0865,
+      "rewards/accuracies": 0.9742187261581421,
+      "rewards/chosen": 2.8573241233825684,
+      "rewards/margins": 4.53125,
+      "rewards/rejected": -1.6725585460662842,
+      "step": 1140
+    },
+    {
+      "epoch": 2.406176393614237,
+      "grad_norm": 6.024884219804737,
+      "learning_rate": 1.1567643011073391e-07,
+      "logits/chosen": -3.496289014816284,
+      "logits/rejected": -3.5318360328674316,
+      "logps/chosen": -289.0249938964844,
+      "logps/rejected": -280.88751220703125,
+      "loss": 0.0834,
+      "rewards/accuracies": 0.983593761920929,
+      "rewards/chosen": 2.84033203125,
+      "rewards/margins": 4.552343845367432,
+      "rewards/rejected": -1.712011694908142,
+      "step": 1150
+    },
+    {
+      "epoch": 2.4271133211201255,
+      "grad_norm": 5.89974463698295,
+      "learning_rate": 1.0800204702439935e-07,
+      "logits/chosen": -3.483203172683716,
+      "logits/rejected": -3.613476514816284,
+      "logps/chosen": -302.1000061035156,
+      "logps/rejected": -273.61248779296875,
+      "loss": 0.0854,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.7640624046325684,
+      "rewards/margins": 4.421093940734863,
+      "rewards/rejected": -1.657958984375,
+      "step": 1160
+    },
+    {
+      "epoch": 2.4480502486260143,
+      "grad_norm": 6.116909678052003,
+      "learning_rate": 1.0056014236546645e-07,
+      "logits/chosen": -3.469531297683716,
+      "logits/rejected": -3.548632860183716,
+      "logps/chosen": -308.9750061035156,
+      "logps/rejected": -280.1499938964844,
+      "loss": 0.0875,
+      "rewards/accuracies": 0.97265625,
+      "rewards/chosen": 2.7728514671325684,
+      "rewards/margins": 4.456640720367432,
+      "rewards/rejected": -1.682519555091858,
+      "step": 1170
+    },
+    {
+      "epoch": 2.4689871761319027,
+      "grad_norm": 5.317406635553003,
+      "learning_rate": 9.335512963221731e-08,
+      "logits/chosen": -3.4771485328674316,
+      "logits/rejected": -3.574023485183716,
+      "logps/chosen": -303.04998779296875,
+      "logps/rejected": -277.17498779296875,
+      "loss": 0.0924,
+      "rewards/accuracies": 0.9703124761581421,
+      "rewards/chosen": 2.782910108566284,
+      "rewards/margins": 4.490429878234863,
+      "rewards/rejected": -1.708642601966858,
+      "step": 1180
+    },
+    {
+      "epoch": 2.489924103637791,
+      "grad_norm": 8.539529378029313,
+      "learning_rate": 8.639128183173516e-08,
+      "logits/chosen": -3.5160155296325684,
+      "logits/rejected": -3.6322264671325684,
+      "logps/chosen": -297.2250061035156,
+      "logps/rejected": -272.7124938964844,
+      "loss": 0.0898,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.707226514816284,
+      "rewards/margins": 4.3857421875,
+      "rewards/rejected": -1.6776854991912842,
+      "step": 1190
+    },
+    {
+      "epoch": 2.51086103114368,
+      "grad_norm": 5.851096093284337,
+      "learning_rate": 7.967272894575311e-08,
+      "logits/chosen": -3.5283203125,
+      "logits/rejected": -3.6058592796325684,
+      "logps/chosen": -286.58123779296875,
+      "logps/rejected": -279.5625,
+      "loss": 0.0895,
+      "rewards/accuracies": 0.97265625,
+      "rewards/chosen": 2.713085889816284,
+      "rewards/margins": 4.398241996765137,
+      "rewards/rejected": -1.684912085533142,
+      "step": 1200
+    },
+    {
+      "epoch": 2.51086103114368,
+      "eval_logits/chosen": -3.479414701461792,
+      "eval_logits/rejected": -3.5473711490631104,
+      "eval_logps/chosen": -295.8254089355469,
+      "eval_logps/rejected": -278.77777099609375,
+      "eval_loss": 0.633702278137207,
+      "eval_rewards/accuracies": 0.65625,
+      "eval_rewards/chosen": 1.0530134439468384,
+      "eval_rewards/margins": 1.234002947807312,
+      "eval_rewards/rejected": -0.1804315447807312,
+      "eval_runtime": 38.0482,
+      "eval_samples_per_second": 52.565,
+      "eval_steps_per_second": 1.656,
+      "step": 1200
+    },
+    {
+      "epoch": 2.5317979586495682,
+      "grad_norm": 5.873497380250258,
+      "learning_rate": 7.320345548132678e-08,
+      "logits/chosen": -3.4486327171325684,
+      "logits/rejected": -3.5687499046325684,
+      "logps/chosen": -290.57501220703125,
+      "logps/rejected": -271.25,
+      "loss": 0.0886,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.6546874046325684,
+      "rewards/margins": 4.319726467132568,
+      "rewards/rejected": -1.663964867591858,
+      "step": 1210
+    },
+    {
+      "epoch": 2.5527348861554566,
+      "grad_norm": 5.273058092571581,
+      "learning_rate": 6.698729810778064e-08,
+      "logits/chosen": -3.4437499046325684,
+      "logits/rejected": -3.599804639816284,
+      "logps/chosen": -288.73748779296875,
+      "logps/rejected": -258.3125,
+      "loss": 0.0809,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.680371046066284,
+      "rewards/margins": 4.344140529632568,
+      "rewards/rejected": -1.662841796875,
+      "step": 1220
+    },
+    {
+      "epoch": 2.5736718136613455,
+      "grad_norm": 7.330712230701304,
+      "learning_rate": 6.102794338133193e-08,
+      "logits/chosen": -3.4312500953674316,
+      "logits/rejected": -3.5152344703674316,
+      "logps/chosen": -301.1000061035156,
+      "logps/rejected": -287.3125,
+      "loss": 0.0951,
+      "rewards/accuracies": 0.9703124761581421,
+      "rewards/chosen": 2.7242674827575684,
+      "rewards/margins": 4.3828125,
+      "rewards/rejected": -1.660009741783142,
+      "step": 1230
+    },
+    {
+      "epoch": 2.594608741167234,
+      "grad_norm": 5.29581218592564,
+      "learning_rate": 5.532892555874058e-08,
+      "logits/chosen": -3.5152344703674316,
+      "logits/rejected": -3.547656297683716,
+      "logps/chosen": -300.9375,
+      "logps/rejected": -285.51873779296875,
+      "loss": 0.0915,
+      "rewards/accuracies": 0.967968761920929,
+      "rewards/chosen": 2.711230516433716,
+      "rewards/margins": 4.498632907867432,
+      "rewards/rejected": -1.787841796875,
+      "step": 1240
+    },
+    {
+      "epoch": 2.615545668673122,
+      "grad_norm": 5.625271423134861,
+      "learning_rate": 4.989362450128132e-08,
+      "logits/chosen": -3.4453125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -302.1499938964844,
+      "logps/rejected": -277.8999938964844,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.772265672683716,
+      "rewards/margins": 4.508593559265137,
+      "rewards/rejected": -1.7361328601837158,
+      "step": 1250
+    },
+    {
+      "epoch": 2.6364825961790106,
+      "grad_norm": 6.7453339198518485,
+      "learning_rate": 4.47252636702829e-08,
+      "logits/chosen": -3.491992235183716,
+      "logits/rejected": -3.5992188453674316,
+      "logps/chosen": -297.01251220703125,
+      "logps/rejected": -272.4375,
+      "loss": 0.0828,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.7337889671325684,
+      "rewards/margins": 4.5,
+      "rewards/rejected": -1.767236351966858,
+      "step": 1260
+    },
+    {
+      "epoch": 2.657419523684899,
+      "grad_norm": 5.733909656340735,
+      "learning_rate": 3.982690821542034e-08,
+      "logits/chosen": -3.4867186546325684,
+      "logits/rejected": -3.5703125,
+      "logps/chosen": -313.1625061035156,
+      "logps/rejected": -282.3125,
+      "loss": 0.084,
+      "rewards/accuracies": 0.9742187261581421,
+      "rewards/chosen": 2.898632764816284,
+      "rewards/margins": 4.657812595367432,
+      "rewards/rejected": -1.7585937976837158,
+      "step": 1270
+    },
+    {
+      "epoch": 2.678356451190788,
+      "grad_norm": 5.951619980133193,
+      "learning_rate": 3.520146315689693e-08,
+      "logits/chosen": -3.4501953125,
+      "logits/rejected": -3.556445360183716,
+      "logps/chosen": -290.0625,
+      "logps/rejected": -269.9125061035156,
+      "loss": 0.0877,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.7286133766174316,
+      "rewards/margins": 4.393359184265137,
+      "rewards/rejected": -1.666015625,
+      "step": 1280
+    },
+    {
+      "epoch": 2.699293378696676,
+      "grad_norm": 6.091215507642081,
+      "learning_rate": 3.085167166259162e-08,
+      "logits/chosen": -3.501757860183716,
+      "logits/rejected": -3.6214842796325684,
+      "logps/chosen": -309.3999938964844,
+      "logps/rejected": -283.04998779296875,
+      "loss": 0.0864,
+      "rewards/accuracies": 0.979687511920929,
+      "rewards/chosen": 2.868359327316284,
+      "rewards/margins": 4.560546875,
+      "rewards/rejected": -1.692529320716858,
+      "step": 1290
+    },
+    {
+      "epoch": 2.7202303062025646,
+      "grad_norm": 8.116516834340125,
+      "learning_rate": 2.6780113421195294e-08,
+      "logits/chosen": -3.4749999046325684,
+      "logits/rejected": -3.5560545921325684,
+      "logps/chosen": -301.3125,
+      "logps/rejected": -281.5375061035156,
+      "loss": 0.0847,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.8023438453674316,
+      "rewards/margins": 4.517382621765137,
+      "rewards/rejected": -1.7168457508087158,
+      "step": 1300
+    },
+    {
+      "epoch": 2.7202303062025646,
+      "eval_logits/chosen": -3.4789185523986816,
+      "eval_logits/rejected": -3.5471229553222656,
+      "eval_logps/chosen": -295.76190185546875,
+      "eval_logps/rejected": -278.73016357421875,
+      "eval_loss": 0.6418492197990417,
+      "eval_rewards/accuracies": 0.6552579402923584,
+      "eval_rewards/chosen": 1.057012677192688,
+      "eval_rewards/margins": 1.2254774570465088,
+      "eval_rewards/rejected": -0.1690228134393692,
+      "eval_runtime": 38.0784,
+      "eval_samples_per_second": 52.523,
+      "eval_steps_per_second": 1.654,
+      "step": 1300
+    },
+    {
+      "epoch": 2.7411672337084534,
+      "grad_norm": 6.921689972824779,
+      "learning_rate": 2.298920311229968e-08,
+      "logits/chosen": -3.451171875,
+      "logits/rejected": -3.5679688453674316,
+      "logps/chosen": -300.9375,
+      "logps/rejected": -277.51251220703125,
+      "loss": 0.0944,
+      "rewards/accuracies": 0.9664062261581421,
+      "rewards/chosen": 2.739453077316284,
+      "rewards/margins": 4.327538967132568,
+      "rewards/rejected": -1.5888183116912842,
+      "step": 1310
+    },
+    {
+      "epoch": 2.7621041612143418,
+      "grad_norm": 5.690860701395228,
+      "learning_rate": 1.9481188974346696e-08,
+      "logits/chosen": -3.4873046875,
+      "logits/rejected": -3.629101514816284,
+      "logps/chosen": -302.95001220703125,
+      "logps/rejected": -269.26251220703125,
+      "loss": 0.0896,
+      "rewards/accuracies": 0.973437488079071,
+      "rewards/chosen": 2.860058546066284,
+      "rewards/margins": 4.494336128234863,
+      "rewards/rejected": -1.634521484375,
+      "step": 1320
+    },
+    {
+      "epoch": 2.78304108872023,
+      "grad_norm": 7.3809685811878545,
+      "learning_rate": 1.6258151471287396e-08,
+      "logits/chosen": -3.440624952316284,
+      "logits/rejected": -3.544140577316284,
+      "logps/chosen": -300.3062438964844,
+      "logps/rejected": -276.71875,
+      "loss": 0.0894,
+      "rewards/accuracies": 0.9742187261581421,
+      "rewards/chosen": 2.844433546066284,
+      "rewards/margins": 4.497265815734863,
+      "rewards/rejected": -1.652929663658142,
+      "step": 1330
+    },
+    {
+      "epoch": 2.803978016226119,
+      "grad_norm": 5.736571953387302,
+      "learning_rate": 1.3322002058741678e-08,
+      "logits/chosen": -3.4009766578674316,
+      "logits/rejected": -3.538281202316284,
+      "logps/chosen": -307.42498779296875,
+      "logps/rejected": -285.0,
+      "loss": 0.0755,
+      "rewards/accuracies": 0.983593761920929,
+      "rewards/chosen": 2.8056640625,
+      "rewards/margins": 4.569726467132568,
+      "rewards/rejected": -1.766845703125,
+      "step": 1340
+    },
+    {
+      "epoch": 2.8249149437320074,
+      "grad_norm": 5.144031930283929,
+      "learning_rate": 1.0674482050389456e-08,
+      "logits/chosen": -3.47265625,
+      "logits/rejected": -3.572070360183716,
+      "logps/chosen": -299.07501220703125,
+      "logps/rejected": -280.76873779296875,
+      "loss": 0.0846,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": 2.8057618141174316,
+      "rewards/margins": 4.482812404632568,
+      "rewards/rejected": -1.677001953125,
+      "step": 1350
+    },
+    {
+      "epoch": 2.8458518712378957,
+      "grad_norm": 6.036457991588913,
+      "learning_rate": 8.317161585266963e-09,
+      "logits/chosen": -3.477734327316284,
+      "logits/rejected": -3.554882764816284,
+      "logps/chosen": -304.73748779296875,
+      "logps/rejected": -279.51251220703125,
+      "loss": 0.0843,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.8153319358825684,
+      "rewards/margins": 4.515234470367432,
+      "rewards/rejected": -1.698974609375,
+      "step": 1360
+    },
+    {
+      "epoch": 2.8667887987437846,
+      "grad_norm": 5.507895293687402,
+      "learning_rate": 6.251438696579292e-09,
+      "logits/chosen": -3.511523485183716,
+      "logits/rejected": -3.594531297683716,
+      "logps/chosen": -292.20001220703125,
+      "logps/rejected": -267.0375061035156,
+      "loss": 0.0881,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.793750047683716,
+      "rewards/margins": 4.418359279632568,
+      "rewards/rejected": -1.624853491783142,
+      "step": 1370
+    },
+    {
+      "epoch": 2.887725726249673,
+      "grad_norm": 5.218809734111947,
+      "learning_rate": 4.478538482582617e-09,
+      "logits/chosen": -3.4720702171325684,
+      "logits/rejected": -3.6068358421325684,
+      "logps/chosen": -306.0,
+      "logps/rejected": -273.57501220703125,
+      "loss": 0.0867,
+      "rewards/accuracies": 0.969531238079071,
+      "rewards/chosen": 2.787792921066284,
+      "rewards/margins": 4.4951171875,
+      "rewards/rejected": -1.707763671875,
+      "step": 1380
+    },
+    {
+      "epoch": 2.9086626537555613,
+      "grad_norm": 7.340549321552548,
+      "learning_rate": 2.999512380027047e-09,
+      "logits/chosen": -3.516406297683716,
+      "logits/rejected": -3.567187547683716,
+      "logps/chosen": -297.3374938964844,
+      "logps/rejected": -274.6625061035156,
+      "loss": 0.0855,
+      "rewards/accuracies": 0.9820312261581421,
+      "rewards/chosen": 2.8055663108825684,
+      "rewards/margins": 4.57421875,
+      "rewards/rejected": -1.7683594226837158,
+      "step": 1390
+    },
+    {
+      "epoch": 2.92959958126145,
+      "grad_norm": 4.847528704619572,
+      "learning_rate": 1.8152375405909304e-09,
+      "logits/chosen": -3.39697265625,
+      "logits/rejected": -3.5400390625,
+      "logps/chosen": -297.2875061035156,
+      "logps/rejected": -274.42498779296875,
+      "loss": 0.0842,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.7293944358825684,
+      "rewards/margins": 4.372460842132568,
+      "rewards/rejected": -1.642187476158142,
+      "step": 1400
+    },
+    {
+      "epoch": 2.92959958126145,
+      "eval_logits/chosen": -3.478670597076416,
+      "eval_logits/rejected": -3.546875,
+      "eval_logps/chosen": -295.77777099609375,
+      "eval_logps/rejected": -278.69842529296875,
+      "eval_loss": 0.638893187046051,
+      "eval_rewards/accuracies": 0.6557539701461792,
+      "eval_rewards/chosen": 1.0641740560531616,
+      "eval_rewards/margins": 1.2395833730697632,
+      "eval_rewards/rejected": -0.1756262332201004,
+      "eval_runtime": 38.342,
+      "eval_samples_per_second": 52.162,
+      "eval_steps_per_second": 1.643,
+      "step": 1400
+    },
+    {
+      "epoch": 2.9505365087673385,
+      "grad_norm": 5.558137624337912,
+      "learning_rate": 9.264163106774137e-10,
+      "logits/chosen": -3.451367139816284,
+      "logits/rejected": -3.5150389671325684,
+      "logps/chosen": -299.625,
+      "logps/rejected": -280.88751220703125,
+      "loss": 0.0787,
+      "rewards/accuracies": 0.985156238079071,
+      "rewards/chosen": 2.7481446266174316,
+      "rewards/margins": 4.5927734375,
+      "rewards/rejected": -1.8439452648162842,
+      "step": 1410
+    },
+    {
+      "epoch": 2.971473436273227,
+      "grad_norm": 6.398175570710044,
+      "learning_rate": 3.335758148803047e-10,
+      "logits/chosen": -3.4453125,
+      "logits/rejected": -3.507031202316284,
+      "logps/chosen": -298.3999938964844,
+      "logps/rejected": -283.1499938964844,
+      "loss": 0.083,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.830859422683716,
+      "rewards/margins": 4.568749904632568,
+      "rewards/rejected": -1.7389647960662842,
+      "step": 1420
+    },
+    {
+      "epoch": 2.9924103637791153,
+      "grad_norm": 7.709725508604469,
+      "learning_rate": 3.70676433677497e-11,
+      "logits/chosen": -3.439257860183716,
+      "logits/rejected": -3.4908204078674316,
+      "logps/chosen": -287.5,
+      "logps/rejected": -283.7749938964844,
+      "loss": 0.0856,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 2.722851514816284,
+      "rewards/margins": 4.5791015625,
+      "rewards/rejected": -1.856103539466858,
+      "step": 1430
+    },
+    {
+      "epoch": 3.0,
+      "step": 1434,
+      "total_flos": 0.0,
+      "train_loss": 0.31465955367480697,
+      "train_runtime": 10878.0316,
+      "train_samples_per_second": 16.86,
+      "train_steps_per_second": 0.132
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1434,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

nemotron_synthetic_1T_exp_7B_DPO/hf/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eca76ef6522aa255ec03f1506f9dac84376b2448079cbb26ff98973878309422
+size 7928