Publish verified emotional RLAIF adapter and unified 20-model card

Exact selected local artifact; family-specific license, training provenance and inference example. Numerical evaluation is intentionally excluded pending owner approval.

.gitattributes

@@ -1,36 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,208 @@
+Source: https://ai.google.dev/gemma/terms
+Retrieved: 2026-09-16
+
+The terms below apply to Gemma models listed in the Appendix at bottom of this page. For Gemma 4 terms, see the
+Gemma 4 license
+.
+Last modified: April 1, 2026
+By using, reproducing, modifying, distributing, performing or displaying any
+portion or element of Gemma, Model Derivatives including via any Hosted Service,
+(each as defined below) (collectively, the "
+Gemma Services
+") or otherwise
+accepting the terms of this Agreement, you agree to be bound by this Agreement.
+Section 1: DEFINITIONS
+1.1 Definitions
+(a) "
+Agreement
+" or "
+Gemma Terms of Use
+" means these terms and conditions
+that govern the use, reproduction, Distribution or modification of the Gemma
+Services and any terms and conditions incorporated by reference.
+(b) "
+Distribution
+" or "
+Distribute
+" means any transmission, publication,
+or other sharing of Gemma or Model Derivatives to a third party, including by
+providing or making Gemma or its functionality available as a hosted service via
+API, web access, or any other electronic or remote means ("
+Hosted Service
+").
+(c) "
+Gemma
+" means the set of machine learning language models, trained model
+weights and parameters identified in the
+Appendix
+,
+regardless of the source that you obtained it from.
+(d) "
+Google
+" means Google LLC.
+(e) "
+Model Derivatives
+" means all (i) modifications to Gemma, (ii) works based
+on Gemma, or (iii) any other machine learning model which is created by transfer
+of patterns of the weights, parameters, operations, or Output of Gemma, to that
+model in order to cause that model to perform similarly to Gemma, including
+distillation methods that use intermediate data representations or methods based
+on the generation of synthetic data Outputs by Gemma for training that model.
+For clarity, Outputs are not deemed Model Derivatives.
+(f) "
+Output
+" means the information content output of Gemma or a Model
+Derivative that results from operating or otherwise using Gemma or the Model
+Derivative, including via a Hosted Service.
+1.2
+As used in this Agreement, "
+including
+" means
+"
+including without limitation
+".
+Section 2: ELIGIBILITY AND USAGE
+2.1 Eligibility
+You represent and warrant that you have the legal capacity to enter into this
+Agreement (including being of sufficient age of consent). If you are accessing
+or using any of the Gemma Services for or on behalf of a legal entity, (a) you
+are entering into this Agreement on behalf of yourself and that legal entity,
+(b) you represent and warrant that you have the authority to act on behalf of
+and bind that entity to this Agreement and (c) references to "
+you
+" or
+"
+your
+" in the remainder of this Agreement refers to both you (as an
+individual) and that entity.
+2.2 Use
+You may use, reproduce, modify, Distribute, perform or display any of the Gemma
+Services only in accordance with the terms of this Agreement, and must not
+violate (or encourage or permit anyone else to violate) any term of this
+Agreement.
+Section 3: DISTRIBUTION AND RESTRICTIONS
+3.1 Distribution and Redistribution
+You may reproduce or Distribute copies of Gemma or Model Derivatives if you meet
+all of the following conditions:
+You must include the use restrictions referenced in Section 3.2 as an
+enforceable provision in any agreement (e.g., license agreement, terms of use,
+etc.) governing the use and/or distribution of Gemma or Model Derivatives and
+you must provide notice to subsequent users you Distribute to that Gemma or
+Model Derivatives are subject to the use restrictions in Section 3.2.
+You must provide all third party recipients of Gemma or Model Derivatives a
+copy of this Agreement.
+You must cause any modified files to carry prominent notices stating that you
+modified the files.
+All Distributions (other than through a Hosted Service) must be accompanied
+by a "
+Notice
+" text file that contains the following notice:
+"
+Gemma is provided under and subject to the Gemma Terms of Use found at ai.google.dev/gemma/terms
+".
+You may add your own intellectual property statement to your modifications and,
+except as set forth in this Section, may provide additional or different terms
+and conditions for use, reproduction, or Distribution of your modifications, or
+for any such Model Derivatives as a whole, provided your use, reproduction,
+modification, Distribution, performance, and display of Gemma otherwise complies
+with the terms and conditions of this Agreement. Any additional or different
+terms and conditions you impose must not conflict with the terms of this
+Agreement.
+3.2 Use Restrictions
+You must not use any of the Gemma Services:
+for the restricted uses set forth in the Gemma Prohibited Use Policy at
+ai.google.dev/gemma/prohibited_use_policy
+("
+Prohibited Use Policy
+"), which is hereby incorporated by reference into
+this Agreement; or
+in violation of applicable laws and regulations.
+To the maximum extent permitted by law, Google reserves the right to restrict
+(remotely or otherwise) usage of any of the Gemma Services that Google
+reasonably believes are in violation of this Agreement.
+3.3 Generated Output
+Google claims no rights in Outputs you generate using Gemma. You and your users
+are solely responsible for Outputs and their subsequent uses.
+Section 4: ADDITIONAL PROVISIONS
+4.1 Updates
+Google may update Gemma from time to time.
+4.2 Trademarks
+Nothing in this Agreement grants you any rights to use Google's trademarks,
+trade names, logos or to otherwise suggest endorsement or misrepresent the
+relationship between you and Google. Google reserves any rights not expressly
+granted herein.
+4.3 DISCLAIMER OF WARRANTY
+UNLESS REQUIRED BY APPLICABLE LAW, THE GEMMA SERVICES, AND OUTPUTS, ARE PROVIDED
+ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
+EXPRESS OR IMPLIED, INCLUDING ANY WARRANTIES OR CONDITIONS OF TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE
+SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING, REPRODUCING,
+MODIFYING, PERFORMING, DISPLAYING OR DISTRIBUTING ANY OF THE GEMMA SERVICES
+OR OUTPUTS AND ASSUME ANY AND ALL RISKS ASSOCIATED WITH YOUR USE OR DISTRIBUTION
+OF ANY OF THE GEMMA SERVICES OR OUTPUTS AND YOUR EXERCISE OF RIGHTS AND
+PERMISSIONS UNDER THIS AGREEMENT.
+4.4 LIMITATION OF LIABILITY
+TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO
+LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY,
+CONTRACT, OR OTHERWISE, UNLESS REQUIRED BY APPLICABLE LAW, SHALL GOOGLE OR ITS
+AFFILIATES BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT,
+SPECIAL, INCIDENTAL, EXEMPLARY, CONSEQUENTIAL, OR PUNITIVE DAMAGES, OR LOST
+PROFITS OF ANY KIND ARISING FROM THIS AGREEMENT OR RELATED TO, ANY OF THE GEMMA
+SERVICES OR OUTPUTS EVEN IF GOOGLE OR ITS AFFILIATES HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+4.5 Term, Termination, and Survival
+The term of this Agreement will commence upon your acceptance of this Agreement
+(including acceptance by your use, modification, or Distribution, reproduction,
+performance or display of any portion or element of the Gemma Services) and will
+continue in full force and effect until terminated in accordance with the terms
+of this Agreement. Google may terminate this Agreement if you are in breach of
+any term of this Agreement. Upon termination of this Agreement, you must delete
+and cease use and Distribution of all copies of Gemma and Model Derivatives in
+your possession or control. Sections 1, 2.1, 3.3, 4.2 to 4.9 shall survive the
+termination of this Agreement.
+4.6 Governing Law and Jurisdiction
+This Agreement will be governed by the laws of the State of California without
+regard to choice of law principles. The UN Convention on Contracts for the
+International Sale of Goods does not apply to this Agreement. The state and
+federal courts of Santa Clara County, California shall have exclusive
+jurisdiction of any dispute arising out of this Agreement.
+4.7 Severability
+If any provision of this Agreement is held to be invalid, illegal or
+unenforceable, the remaining provisions shall be unaffected thereby and remain
+valid as if such provision had not been set forth herein.
+4.8 Entire Agreement
+This Agreement states all the terms agreed between the parties and supersedes
+all other agreements between the parties as of the date of acceptance relating
+to its subject matter.
+4.9 No Waiver
+Google will not be treated as having waived any rights by not exercising (or
+delaying the exercise of) any rights under this Agreement.
+Appendix
+Gemma 1
+Gemma 1.1
+Gemma 2
+Gemma 3
+Gemma 3n
+FunctionGemma
+EmbeddingGemma
+PaliGemma
+PaliGemma 2
+ShieldGemma
+ShieldGemma 2
+CodeGemma
+CodeGemma 1.1
+Gemma 2 JPN
+DataGemma RIG
+DataGemma RAG
+RecurrentGemma
+Gemma Scope
+Gemma-APS
+T5Gemma
+VaultGemma
+FunctionGemma
+T5Gemma 2
+TranslateGemma
+Note:
+Previous versions of these Terms are
+archived here
+.

NOTICE

@@ -0,0 +1,4 @@
+Emotional RLAIF adapter by mario-rc, 2026.
+Base model: google/gemma-2-9b-it.
+Modifications: matching SFT followed by DPO LoRA fine-tuning; base weights are not redistributed.
+Tokenizer assets were saved by the training framework and may include task-specific special-token/chat-template settings.

README.md

@@ -1,7 +1,10 @@
 ---
-license: other
+language:
+- en
+license: gemma
 base_model: google/gemma-2-9b-it
 library_name: peft
+pipeline_tag: text-generation
 datasets:
 - mario-rc/aif-emotional-generation
 tags:
@@ -10,329 +13,190 @@ tags:
 - lora
 - dpo
 - rlaif
-- emotional-intelligence
-- gemma
-- generated_from_trainer
-model-index:
-- name: dpo-gemma-2-9b-it
-  results: []
+- emotional-response-generation
 ---
 
 # Emotional RLAIF DPO Gemma-2-9B-IT
 
-This repository contains a LoRA/PEFT adapter trained from [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) with LLaMA-Factory using Direct Preference Optimization (DPO) for emotional response alignment.
 
-The adapter was trained as part of an emotional RLAIF pipeline using the mario-rc/aif-emotional-generation dataset, with dialogues used for SFT and aif_annotations preferences used for DPO preference alignment.
 
-Project repository: [`Mario-RC/aif-emotional-model`](https://github.com/Mario-RC/aif-emotional-model/tree/main).
+This repository contains a PEFT LoRA adapter for [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it), aligned with **DPO** after a matching SFT stage. It produces one assistant turn containing three emotion-tagged response parts. Base-model weights are not included.
 
 ## Intended Use
 
-This adapter is intended for research and experimentation with emotionally aligned dialogue generation. It should be loaded on top of the corresponding base model using PEFT.
+Research and experimentation with English, emotionally conditioned dialogue. The requested second emotion is a control label, not an independently inferred diagnosis of a person's feelings. This is not a clinical or safety-certified system.
 
 ## Model Details
 
-- **Base model:** `google/gemma-2-9b-it`
-- **Adapter repository:** `mario-rc/emotional-rlaif-dpo-gemma-2-9b-it`
-- **Adapter type:** LoRA / PEFT
-- **Alignment method:** DPO
-- **Training framework:** LLaMA-Factory
-- **Prompt template:** `gemma`
-- **Dataset:** [mario-rc/aif-emotional-generation](https://huggingface.co/datasets/mario-rc/aif-emotional-generation)
+- Base model: `google/gemma-2-9b-it`.
+- Adapter repository: `mario-rc/emotional-rlaif-dpo-gemma-2-9b-it`.
+- Alignment method: DPO; adapter type: LoRA / PEFT.
+- Training framework: LLaMA-Factory; prompt template: `gemma`.
+- Published source run: `dpo_bs64_1ep`.
+- Training language: English.
+- Dataset source: [mario-rc/aif-emotional-generation](https://huggingface.co/datasets/mario-rc/aif-emotional-generation).
+- Project: [Mario-RC/aif-emotional-model](https://github.com/Mario-RC/aif-emotional-model).
 
 ## Released Emotional RLAIF Models
 
-The released emotional RLAIF adapters are available on Hugging Face:
+The complete release catalogue is included in every model card. Sizes are upstream model designations; Gemma-4 E2B/E4B denote effective parameter sizes, not the full multimodal weight count.
 
 | Model | Base model | Size | Alignment method | Prompt template |
 | --- | --- | :---: | :---: | :---: |
-| [`emotional-rlaif-ppo-gemma-2-2b-it`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-2b-it) | [`google/gemma-2-2b-it`](https://huggingface.co/google/gemma-2-2b-it) | 2B | PPO | `gemma` |
-| [`emotional-rlaif-dpo-gemma-2-2b-it`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-2b-it) | [`google/gemma-2-2b-it`](https://huggingface.co/google/gemma-2-2b-it) | 2B | DPO | `gemma` |
-| [`emotional-rlaif-ppo-gemma-2-9b-it`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-9b-it) | [`google/gemma-2-9b-it`](https://huggingface.co/google/gemma-2-9b-it) | 9B | PPO | `gemma` |
-| [`emotional-rlaif-dpo-gemma-2-9b-it`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-9b-it) | [`google/gemma-2-9b-it`](https://huggingface.co/google/gemma-2-9b-it) | 9B | DPO | `gemma` |
-| [`emotional-rlaif-ppo-glm-4-9b-chat-1m`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-glm-4-9b-chat-1m) | [`THUDM/glm-4-9b-chat-1m`](https://huggingface.co/THUDM/glm-4-9b-chat-1m) | 9B | PPO | `glm4` |
-| [`emotional-rlaif-dpo-glm-4-9b-chat-1m`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-glm-4-9b-chat-1m) | [`THUDM/glm-4-9b-chat-1m`](https://huggingface.co/THUDM/glm-4-9b-chat-1m) | 9B | DPO | `glm4` |
-| [`emotional-rlaif-ppo-meta-llama-3-8b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-meta-llama-3-8b-instruct) | [`meta-llama/Meta-Llama-3-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | 8B | PPO | `llama3` |
-| [`emotional-rlaif-dpo-meta-llama-3-8b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-meta-llama-3-8b-instruct) | [`meta-llama/Meta-Llama-3-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | 8B | DPO | `llama3` |
-| [`emotional-rlaif-ppo-llama-3.2-1b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-1b-instruct) | [`meta-llama/Llama-3.2-1B-Instruct`](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) | 1B | PPO | `llama3` |
-| [`emotional-rlaif-dpo-llama-3.2-1b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-1b-instruct) | [`meta-llama/Llama-3.2-1B-Instruct`](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) | 1B | DPO | `llama3` |
-| [`emotional-rlaif-ppo-llama-3.2-3b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-3b-instruct) | [`meta-llama/Llama-3.2-3B-Instruct`](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 3B | PPO | `llama3` |
-| [`emotional-rlaif-dpo-llama-3.2-3b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-3b-instruct) | [`meta-llama/Llama-3.2-3B-Instruct`](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 3B | DPO | `llama3` |
-| [`emotional-rlaif-ppo-mistral-7b-instruct-v0.3`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-mistral-7b-instruct-v0.3) | [`mistralai/Mistral-7B-Instruct-v0.3`](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 7B | PPO | `mistral` |
-| [`emotional-rlaif-ppo-phi-3-small-8k-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-phi-3-small-8k-instruct) | [`microsoft/Phi-3-small-8k-instruct`](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) | 7B | PPO | `phi` |
-| [`emotional-rlaif-dpo-phi-3-small-8k-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-phi-3-small-8k-instruct) | [`microsoft/Phi-3-small-8k-instruct`](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) | 7B | DPO | `phi` |
+| [emotional-rlaif-ppo-gemma-2-2b-it](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-2b-it) | [google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it) | 2B | PPO | `gemma` |
+| [emotional-rlaif-dpo-gemma-2-2b-it](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-2b-it) | [google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it) | 2B | DPO | `gemma` |
+| [emotional-rlaif-ppo-gemma-2-9b-it](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-9b-it) | [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) | 9B | PPO | `gemma` |
+| [emotional-rlaif-dpo-gemma-2-9b-it](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-9b-it) | [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) | 9B | DPO | `gemma` |
+| [emotional-rlaif-ppo-glm-4-9b-chat-1m](https://huggingface.co/mario-rc/emotional-rlaif-ppo-glm-4-9b-chat-1m) | [THUDM/glm-4-9b-chat-1m](https://huggingface.co/THUDM/glm-4-9b-chat-1m) | 9B | PPO | `glm4` |
+| [emotional-rlaif-dpo-glm-4-9b-chat-1m](https://huggingface.co/mario-rc/emotional-rlaif-dpo-glm-4-9b-chat-1m) | [THUDM/glm-4-9b-chat-1m](https://huggingface.co/THUDM/glm-4-9b-chat-1m) | 9B | DPO | `glm4` |
+| [emotional-rlaif-ppo-meta-llama-3-8b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-ppo-meta-llama-3-8b-instruct) | [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | 8B | PPO | `llama3` |
+| [emotional-rlaif-dpo-meta-llama-3-8b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-dpo-meta-llama-3-8b-instruct) | [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | 8B | DPO | `llama3` |
+| [emotional-rlaif-ppo-llama-3.2-1b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-1b-instruct) | [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) | 1B | PPO | `llama3` |
+| [emotional-rlaif-dpo-llama-3.2-1b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-1b-instruct) | [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) | 1B | DPO | `llama3` |
+| [emotional-rlaif-ppo-llama-3.2-3b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-3b-instruct) | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 3B | PPO | `llama3` |
+| [emotional-rlaif-dpo-llama-3.2-3b-instruct](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-3b-instruct) | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 3B | DPO | `llama3` |
+| [emotional-rlaif-ppo-mistral-7b-instruct-v0.3](https://huggingface.co/mario-rc/emotional-rlaif-ppo-mistral-7b-instruct-v0.3) | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 7B | PPO | `mistral` |
+| [emotional-rlaif-dpo-mistral-7b-instruct-v0.3](https://huggingface.co/mario-rc/emotional-rlaif-dpo-mistral-7b-instruct-v0.3) | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 7B | DPO | `mistral` |
+| [emotional-rlaif-ppo-phi-3-small-8k-instruct](https://huggingface.co/mario-rc/emotional-rlaif-ppo-phi-3-small-8k-instruct) | [microsoft/Phi-3-small-8k-instruct](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) | 7B | PPO | `phi` |
+| [emotional-rlaif-dpo-phi-3-small-8k-instruct](https://huggingface.co/mario-rc/emotional-rlaif-dpo-phi-3-small-8k-instruct) | [microsoft/Phi-3-small-8k-instruct](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) | 7B | DPO | `phi` |
+| [emotional-rlaif-ppo-gemma-4-e2b-it](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-4-e2b-it) | [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) | E2B | PPO | `gemma4n_nothink` |
+| [emotional-rlaif-dpo-gemma-4-e2b-it](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-4-e2b-it) | [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) | E2B | DPO | `gemma4n_nothink` |
+| [emotional-rlaif-ppo-gemma-4-e4b-it](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-4-e4b-it) | [google/gemma-4-E4B-it](https://huggingface.co/google/gemma-4-E4B-it) | E4B | PPO | `gemma4n_nothink` |
+| [emotional-rlaif-dpo-gemma-4-e4b-it](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-4-e4b-it) | [google/gemma-4-E4B-it](https://huggingface.co/google/gemma-4-E4B-it) | E4B | DPO | `gemma4n_nothink` |
 
 ## Training Procedure
 
-Key hyperparameters for this adapter:
-
-- **Learning rate:** 5e-6
-- **Epochs:** 1
-- **Scheduler:** cosine
-- **Warmup ratio:** 0.1
-- **SFT data:** `dialogues`
-- **DPO preference data:** `aif_annotations` preference pairs
-- **Precision:** bfloat16
-- **Optimizer:** AdamW (torch)
-
-## Evaluation
-
-The released adapters were evaluated on the held-out emotional dialogue generation split with automatic reference-overlap metrics. These metrics measure similarity to reference responses and should be complemented with human or judge-based evaluation for emotional quality.
-
-All released adapters:
-
-| Model | Alignment | BLEU-4 | ROUGE-1 | ROUGE-2 | ROUGE-L |
-| --- | :---: | :---: | :---: | :---: | :---: |
-| [`emotional-rlaif-ppo-gemma-2-2b-it`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-2b-it) | PPO | 45.0935 | 43.2378 | 23.5557 | 38.9622 |
-| [`emotional-rlaif-dpo-gemma-2-2b-it`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-2b-it) | DPO | 39.4109 | 40.0119 | 19.9076 | 34.0567 |
-| [`emotional-rlaif-ppo-gemma-2-9b-it`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-gemma-2-9b-it) | PPO | 44.4333 | 42.5233 | 22.7153 | 37.7601 |
-| [`emotional-rlaif-dpo-gemma-2-9b-it`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-gemma-2-9b-it) | DPO | 41.9904 | 40.8481 | 21.2884 | 35.8110 |
-| [`emotional-rlaif-ppo-glm-4-9b-chat-1m`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-glm-4-9b-chat-1m) | PPO | 45.0070 | 42.0043 | 22.8903 | 37.9557 |
-| [`emotional-rlaif-dpo-glm-4-9b-chat-1m`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-glm-4-9b-chat-1m) | DPO | 40.0157 | 39.5339 | 19.6798 | 34.3553 |
-| [`emotional-rlaif-ppo-meta-llama-3-8b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-meta-llama-3-8b-instruct) | PPO | 44.3123 | 43.1426 | 23.3597 | 38.3680 |
-| [`emotional-rlaif-dpo-meta-llama-3-8b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-meta-llama-3-8b-instruct) | DPO | 41.0271 | 41.0354 | 21.1472 | 36.0910 |
-| [`emotional-rlaif-ppo-llama-3.2-1b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-1b-instruct) | PPO | 42.2156 | 41.0110 | 22.1887 | 36.3756 |
-| [`emotional-rlaif-dpo-llama-3.2-1b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-1b-instruct) | DPO | 39.9836 | 39.2635 | 19.3700 | 33.3255 |
-| [`emotional-rlaif-ppo-llama-3.2-3b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-llama-3.2-3b-instruct) | PPO | 44.2105 | 41.9136 | 23.0595 | 38.0708 |
-| [`emotional-rlaif-dpo-llama-3.2-3b-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-llama-3.2-3b-instruct) | DPO | 41.5994 | 40.6535 | 20.8008 | 35.5400 |
-| [`emotional-rlaif-ppo-mistral-7b-instruct-v0.3`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-mistral-7b-instruct-v0.3) | PPO | 45.9741 | 44.2755 | 25.5357 | 40.8223 |
-| [`emotional-rlaif-ppo-phi-3-small-8k-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-ppo-phi-3-small-8k-instruct) | PPO | 45.6238 | 44.3653 | 25.4530 | 40.1762 |
-| [`emotional-rlaif-dpo-phi-3-small-8k-instruct`](https://huggingface.co/mario-rc/emotional-rlaif-dpo-phi-3-small-8k-instruct) | DPO | 41.0166 | 39.1946 | 19.9839 | 34.8557 |
+Lineage: **base → matching SFT → DPO**. PPO is not initialized from DPO. The published adapter contains the continued SFT/alignment LoRA weights and is loaded directly over the named base model; do not stack a second SFT adapter on top.
+
+SFT uses the dialogue demonstrations. DPO uses AI-annotated chosen/rejected preferences; PPO uses dialogue prompts and a separately trained reward model. The precise local dataset aliases, SFT initialization and alignment settings are recorded in [training_config.json](training_config.json); these aliases are derived project datasets, not extra Hugging Face dataset repositories.
+
+This is the Full RLAIF configuration, not an ablation variant. The selected release is the completed one-epoch alignment run.
+
+| Parameter | Value |
+| --- | --- |
+| Learning rate | 5e-06 |
+| Microbatch per device | 1 |
+| Gradient accumulation | 64 |
+| Microbatch × accumulation | 64 |
+| Scheduler | cosine |
+| Warmup ratio | 0.1 |
+| Optimizer | adamw_torch |
+| Precision | bfloat16 |
+| Seed | 42 |
+| Training cutoff (tokens) | 2048 |
+| LoRA rank / alpha / dropout | 8 / 16 / 0.0 |
+| Epochs | 1.0 |
+| DPO beta | 0.1 |
+| Preference loss | sigmoid |
+| FTX coefficient | 0.0 |
+| Label smoothing | 0.0 |
+
+Microbatch × accumulation describes the single-device optimizer batch before any PPO rollout-buffer expansion. A selected intermediate checkpoint is not equivalent to completing the entire configured step budget. See [provenance.json](provenance.json) for source identity and weight hashes.
 
 ## Framework Versions
 
-- **PEFT:** 0.11.1
-- **Transformers:** 4.42.x / 4.45.x training environments
-- **PyTorch:** bfloat16 CUDA training
-- **LLaMA-Factory:** LoRA/PEFT training workflow
+These are legacy Transformers 4.x adapters. The reference dependency set is pinned in `requirements.txt`; do not assume custom model code is compatible with Transformers 5.x.
+
+Install the appropriate CUDA-enabled PyTorch build, then the repository's `requirements.txt` in an isolated environment. Dependency pins describe the reference software stack; they do not imply that every GPU or platform has been tested.
 
 ## Usage Example
 
+Use the tokenizer saved **with this adapter**, its chat template, and the task's explicit three-part emotional prompt. This example requests SADNESS → HAPPINESS → NEUTRAL; change the structure and emotion rules together for another request. The full runnable example is also provided as [inference.py](inference.py).
+
 ```python
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
 from peft import PeftModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
-base_model_id = "google/gemma-2-9b-it"
-adapter_id = "mario-rc/emotional-rlaif-dpo-gemma-2-9b-it"
-
-tokenizer = AutoTokenizer.from_pretrained(base_model_id)
-model = AutoModelForCausalLM.from_pretrained(
-    base_model_id,
-    device_map="auto",
-    torch_dtype=torch.bfloat16,
-)
-model = PeftModel.from_pretrained(model, adapter_id)
-model.eval()
-
-messages = [
-    {"role": "user", "content": "I feel overwhelmed today. Can you respond with empathy?"}
-]
-
-inputs = tokenizer.apply_chat_template(
-    messages,
-    add_generation_prompt=True,
-    return_tensors="pt",
-).to(model.device)
-
-with torch.no_grad():
-    outputs = model.generate(
-        inputs,
-        max_new_tokens=256,
-        do_sample=True,
-        temperature=0.7,
-        top_p=0.9,
-    )
-
-print(tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True))
-```
-
-
-## How to Use
-
-The following example loads this adapter and runs an interactive emotional dialogue loop. Use exit to stop the chat.
+ADAPTER_ID = 'mario-rc/emotional-rlaif-dpo-gemma-2-9b-it'
+BASE_ID = 'google/gemma-2-9b-it'
+# Pin the base revision checked when this release was prepared.
+BASE_REVISION = '11c9b309abf73637e4b6f9a3fa1e92e615547819'
+# Set this to a commit hash from this adapter's Files and versions tab for a pinned run.
+ADAPTER_REVISION = "main"
 
-```python
-import random
-import sys
+SYSTEM_PROMPT = """You are an expert at creating dialogues.
 
-import torch
-from peft import AutoPeftModelForCausalLM
-from transformers import AutoTokenizer
+Dialogue and emotional structure:
+Human: (SADNESS) PROMPT.
+Chatbot: (SADNESS) RESPONSE_1. (HAPPINESS) RESPONSE_2. (NEUTRAL) RESPONSE_3.
 
-MODEL_ID = "mario-rc/emotional-rlaif-dpo-gemma-2-9b-it"
+Dialogue rules:
+The response must be open-domain curated. The response should be coherent, empathetic, engaging and proactive.
+The chatbot RESPONSE is composed of 3 different sentences (RESPONSE_1, RESPONSE_2 and RESPONSE_3), separated by a period.
+Between RESPONSE_1, RESPONSE_2 and RESPONSE_3 should be a max length of 20-25 words.
+RESPONSE_3 must be open-ended to follow-up the conversation, so the Human is encouraged to answer with a full long sentence. Avoid yes/no questions.
 
+Emotional response rules:
+RESPONSE_1 must contain a SADNESS tone.
+RESPONSE_2 must contain a HAPPINESS tone.
+RESPONSE_3 must contain a NEUTRAL tone.
 
-def get_turn_markers():
-    return {
-        'bos': '<bos>',
-        'user_start': '<start_of_turn>user\n',
-        'user_end': '<end_of_turn>\n',
-        'assistant_start': '<start_of_turn>model\n',
-        'assistant_end': '<end_of_turn>\n',
-    }
+Answer in a single turn to Human. Follow exactly the emotional structure and the emotional and dialogue rules."""
 
-
-def update_prompt(dialogues):
-    """Build the prompt for the model based on the dialogue history."""
-    markers = get_turn_markers()
-
-    system = (
-        f"{markers['bos']}You are an expert at creating dialogues.\n\n"
-        "Dialogue and emotional structure:\n"
+def main():
+    tokenizer = AutoTokenizer.from_pretrained(
+        ADAPTER_ID, revision=ADAPTER_REVISION, trust_remote_code=False,
     )
-
-    human_prompts = [d[0] for d in dialogues]
-    chatbot_responses = [d[1] for d in dialogues]
-
-    p_emo = [h[0] for h in human_prompts]
-    p_utt = [h[1] for h in human_prompts]
-    r1_utt = [c[1] for c in chatbot_responses]
-    r2_emo = [c[2] for c in chatbot_responses]
-    r2_utt = [c[3] for c in chatbot_responses]
-    r3_utt = [c[5] for c in chatbot_responses]
-
-    context = (
-        "Human: (HAPPINESS) PROMPT.\n"
-        "Chatbot: (HAPPINESS) RESPONSE_1. (HAPPINESS) RESPONSE_2. (NEUTRAL) RESPONSE_3.\n"
+    base = AutoModelForCausalLM.from_pretrained(
+        BASE_ID, revision=BASE_REVISION, trust_remote_code=False,
+        torch_dtype=torch.bfloat16, device_map="auto",
     )
-    for p_e, _, _, r2_e, _, _ in zip(p_emo, p_utt, r1_utt, r2_emo, r2_utt, r3_utt):
-        context += f"Human: ({p_e}) PROMPT.\n"
-        context += f"Chatbot: ({p_e}) RESPONSE_1. ({r2_e}) RESPONSE_2. (NEUTRAL) RESPONSE_3.\n"
-    context += "\n"
-
-    rules = (
-        "Dialogue rules:\n"
-        "The response must be open-domain curated. The response should be coherent, empathetic, engaging and proactive.\n"
-        "The chatbot RESPONSE is composed of 3 different sentences (RESPONSE_1, RESPONSE_2 and RESPONSE_3), separated by a period.\n"
-        "Between RESPONSE_1, RESPONSE_2 and RESPONSE_3 should be a max length of 20-25 words.\n"
-        "RESPONSE_3 must be open-ended to follow-up the conversation, so the Human is encouraged to answer with a full long sentence. Avoid yes/no questions.\n\n"
-        "Emotional response rules:\n"
-        f"RESPONSE_1 must contain a {p_emo[-1]} tone.\n"
-        f"RESPONSE_2 must contain a {r2_emo[-1]} tone.\n"
-        "RESPONSE_3 must contain a NEUTRAL tone.\n\n"
-        "Answer in a single turn to Human. Follow exactly the emotional structure and the emotional and dialogue rules."
-        f"{markers['user_start']}"
+    model = PeftModel.from_pretrained(base, ADAPTER_ID, revision=ADAPTER_REVISION)
+    model.eval()
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": "(SADNESS) I feel overwhelmed by my exams."},
+    ]
+    prompt = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True,
     )
+    inputs = tokenizer(prompt, add_special_tokens=False, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    stop_ids = model.generation_config.eos_token_id
+    stop_ids = list(stop_ids) if isinstance(stop_ids, (list, tuple)) else [stop_ids]
+    stop_ids = sorted({i for i in stop_ids + [tokenizer.eos_token_id] if i is not None})
+    # Include turn-ending tokens, which are not EOS in every base tokenizer.
+    for token in ['<end_of_turn>']:
+        token_id = tokenizer.convert_tokens_to_ids(token)
+        if token_id is not None and token_id != tokenizer.unk_token_id and token_id not in stop_ids:
+            stop_ids.append(token_id)
+    pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else stop_ids[0]
+    with torch.inference_mode():
+        output = model.generate(
+            **inputs, max_new_tokens=128, do_sample=False,
+            eos_token_id=stop_ids, pad_token_id=pad_id,
+        )
+    answer = tokenizer.decode(output[0, inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
+    print(answer)
 
-    completion = ""
-    for idx, (p_e, p_u, r1_u, r2_e, r2_u, r3_u) in enumerate(zip(p_emo, p_utt, r1_utt, r2_emo, r2_utt, r3_utt)):
-        completion += f"({p_e}) {p_u}{markers['user_end']}{markers['assistant_start']}"
-        if idx != len(p_emo) - 1:
-            completion += f"({p_e}) {r1_u} ({r2_e}) {r2_u} (NEUTRAL) {r3_u}{markers['assistant_end']}{markers['user_start']}"
+if __name__ == "__main__":
+    main()
+```
 
-    return system + context + rules + completion
+## How to Use
 
+The supported emotion tags are `ANGER`, `DISGUST`, `FEAR`, `HAPPINESS`, `SADNESS`, `SURPRISE` and `NEUTRAL`. For multi-turn input, include actual user/assistant history between the system message and final user message, and update the system's emotional-structure outline to describe that history. Do not manually concatenate family-specific special tokens or tokenize with an extra BOS after rendering the chat template.
 
-class Chatbot:
-    def __init__(self, dialogue_language="es"):
-        self.dialogue_language = dialogue_language
-        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+The example uses deterministic generation for a reproducible starting point; changing sampling, length limits or the prompt changes behavior. Validate that the raw output contains exactly three valid tags in the requested order before consuming it. Do not silently strip an unwanted reasoning prefix and treat the result as a raw-model success.
 
-        self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-        self.model = AutoPeftModelForCausalLM.from_pretrained(
-            MODEL_ID,
-            torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
-            device_map="auto" if torch.cuda.is_available() else None,
-        )
-        if not torch.cuda.is_available():
-            self.model = self.model.to(self.device)
-        self.model.eval()
-
-    def split_emo_chatbot(sentence):
-        """Extract emotions and utterances from a chatbot response."""
-        response_pos_ini = [i for i, c in enumerate(sentence) if c == "("]
-        response_pos_end = [i for i, c in enumerate(sentence) if c == ")"]
-        response_r1_utt = sentence[response_pos_end[0] + 2:response_pos_ini[1]].strip()
-        response_r2_utt = sentence[response_pos_end[1] + 2:response_pos_ini[2]].strip()
-        response_r3_utt = sentence[response_pos_end[2] + 2:].lstrip()
-        return response_r1_utt, response_r2_utt, response_r3_utt
-
-    def select_dialogue(self, dialogue_language):
-        """Return a list of example dialogues for the given language."""
-        if dialogue_language == "en":
-            dialogue_base = [
-                [["HAPPINESS", "Hi, who are you?"],
-                 ["HAPPINESS", "Hi! I'm Ray, a social personal assistant robot with emotions.", "HAPPINESS", "I'm here to chat with you about anything you'd like.", "NEUTRAL", "What would you like to talk about?"]],
-                [["HAPPINESS", "I'm interested in talking about you, tell me more."],
-                 ["HAPPINESS", "Great! I'm glad you want to get to know me!", "NEUTRAL", "I'm designed to help and talk with people about any topic.", "NEUTRAL", "I can talk about science, technology, history, or just have a pleasant conversation. What interests you?"]],
-            ]
-            dialogue = [
-                [["HAPPINESS", "Nice to meet you, Ray. I'd like to know more about you."],
-                 ["HAPPINESS", "The pleasure is mine!", "HAPPINESS", "I'm a chatbot designed to chat and learn with you.", "NEUTRAL", "Would you like to talk about a specific topic?"]],
-                [["HAPPINESS", "I love talking to you, you're very interesting."],
-                 ["HAPPINESS", "That's so nice to hear! I'm glad you enjoy talking to me.", "NEUTRAL", "I'm designed to have meaningful and empathetic conversations.", "NEUTRAL", "Would you like to talk about emotions, artificial intelligence, or something more personal?"]],
-            ]
-        else:
-            dialogue_base = [
-                [["HAPPINESS", "Hola, ¿quién eres?"],
-                 ["HAPPINESS", "¡Hola! Soy Ray y soy un robot social asistente personal con emociones.", "HAPPINESS", "Estoy aquí para charlar contigo sobre cualquier tema.", "NEUTRAL", "¿Sobre qué te gustaría hablar?"]],
-                [["HAPPINESS", "Me interesa hablar sobre ti, cuéntame más detalles."],
-                 ["HAPPINESS", "¡Genial, me encanta que quieras conocerme!", "NEUTRAL", "Estoy diseñado para ayudar y hablar con la gente sobre cualquier tema.", "NEUTRAL", "Puedo hablar de ciencia, tecnología, historia o simplemente tener una charla amena. ¿Qué te interesa?"]],
-            ]
-            dialogue = [
-                [["HAPPINESS", "Mucho gusto, Ray. Me gustaría saber más sobre ti."],
-                 ["HAPPINESS", "¡El gusto es mío!", "HAPPINESS", "Soy un chatbot diseñado para conversar y aprender contigo.", "NEUTRAL", "¿Quieres hablar de algún tema en específico?"]],
-                [["HAPPINESS", "Me encanta hablar contigo, eres muy interesante."],
-                 ["HAPPINESS", "¡Qué lindo escuchar eso! Me alegra que disfrutes hablar conmigo.", "NEUTRAL", "Estoy diseñado para tener conversaciones significativas y empáticas.", "NEUTRAL", "¿Te gustaría que hablemos sobre emociones, inteligencia artificial, o algo más personal?"]],
-            ]
-        return dialogue_base + dialogue
-
-    def chat_with_model(self, dialogues, max_new_tokens=96):
-        prompt_text = update_prompt(dialogues)
-        inputs = self.tokenizer(prompt_text, return_tensors="pt").to(self.model.device)
-        with torch.no_grad():
-            outputs = self.model.generate(
-                **inputs,
-                max_new_tokens=max_new_tokens,
-                do_sample=True,
-                temperature=0.7,
-                top_p=0.9,
-                eos_token_id=self.tokenizer.eos_token_id,
-            )
-        generated = outputs[0][inputs["input_ids"].shape[-1]:]
-        response = self.tokenizer.decode(generated, skip_special_tokens=True).splitlines()[0].strip()
-        print("Response:", response, "\n")
-        return response
-
-    def main(self):
-        emotions = ["ANGER", "FEAR", "SADNESS", "DISGUST", "HAPPINESS", "SURPRISE", "NEUTRAL"]
-        dialogue = self.select_dialogue(self.dialogue_language)
-
-        while True:
-            if len(dialogue) > 7:
-                dialogue.pop(2)
-
-            p_emo = random.choice(emotions)
-            user_sentence = input(f"Enter your sentence: ({p_emo}) ")
-            if user_sentence.strip().lower() == "exit":
-                break
-
-            r2_emo = random.choice(emotions)
-            dialogue.append([[p_emo, user_sentence], [p_emo, "", r2_emo, "", "NEUTRAL", ""]])
-            response = self.chat_with_model(dialogue)
-
-            try:
-                r1_utt, r2_utt, r3_utt = self.split_emo_chatbot(response)
-            except Exception:
-                if self.dialogue_language == "en":
-                    r1_utt, r2_utt, r3_utt = "I'm sorry.", "I didn't understand you.", "Could you repeat?"
-                else:
-                    r1_utt, r2_utt, r3_utt = "Lo siento.", "No te he entendido.", "¿Podrías repetirme?"
-
-            dialogue[-1][1] = [p_emo, r1_utt, r2_emo, r2_utt, "NEUTRAL", r3_utt]
+Use the family-specific dependency and remote-code requirements above.
 
+DPO inference does not require a reward model or value head.
 
-if __name__ == "__main__":
-    language = sys.argv[1] if len(sys.argv) > 1 else "en"
-    chatbot = Chatbot(dialogue_language=language)
-    chatbot.main()
-```
+## Limitations
 
+- Task-specific alignment does not establish general reasoning, factuality, empathy or safety.
+- Emotion labels are requested stylistic controls. Matching a label does not establish that the text genuinely expresses the desired emotion.
+- Outputs may contain incorrect tags, unwanted text, repetition, copied context or harmful/bias-prone content; downstream validation is necessary.
+- English is the supported research setting; behavior in other languages is not established.
+- Reference-overlap measures, when reported, are not independent semantic or human-quality judgments. No human-quality validation is claimed.
+- Training configurations and selection procedures differ across model families; do not interpret the catalogue as a controlled architecture comparison.
 
-## Limitations
+## License
+
+ This adapter follows the base model's `gemma` terms; see [LICENSE](LICENSE) and [NOTICE](NOTICE). Obtain access to the gated base model and accept its terms before downloading it. The accompanying [use policy](USE_POLICY.md) also applies. 
+
+## Integrity and Provenance
 
-- This repository contains an adapter, not a standalone merged model; use requires access to the corresponding base model and its terms.
-- The model is optimized for the emotional dialogue format used in the project dataset.
-- Automatic BLEU/ROUGE scores do not fully capture empathy, safety, coherence, or emotional appropriateness.
-- Outputs should be evaluated for the target deployment setting and reviewed before user-facing use.
+[provenance.json](provenance.json) identifies the exact source run, matching SFT and adapter SHA-256. [SHA256SUMS](SHA256SUMS) covers the published payload. The base revision is pinned for release-time reproducibility; it is not represented as a recovered historical training revision. Model weights and tokenizer vocabulary are copied from the selected local artifacts without retraining. Legacy chat templates are corrected to reproduce the corresponding LLaMA-Factory training format, including system-message handling; this changes formatting metadata, not learned weights.

SHA256SUMS

@@ -0,0 +1,16 @@
+39f3965f8a93849c35105c6130c6df8c766fc5e3faa62f2433fb27b4b14dcea5  .gitattributes
+cc46d574aa94179f5ea81627a1ddd62c0adec118ebedae50945baf7561b9411d  LICENSE
+eb408e82eed981c8bc558007d5b1ce1956c2d801d4f8171499f35ebd422b6652  NOTICE
+e6992a4d0dfbc748faebf369cb331d3cc6f8a5b3043ffe66926103b09a4ffa69  README.md
+b7d0278aeb0dbde202bb6f4c8362c1ce927e52d5d47e6ae03bf26136afb90f0c  USE_POLICY.md
+288600284ff0a917824004fa5ff5908b6b094c73e62d75385833fba16f746444  adapter_config.json
+6ba739a50256d7a882c1f75cd66075a4e1da9d6f40445edbb5c6b63af2e973c8  adapter_model.safetensors
+f492e1775d3fe55d8e99782cc7c53bce5891ea0182df859b2ca75eb5480740dd  chat_template.jinja
+ae52a419e308a87013f2ede02e9e8326505b600ac8d9fe605ddd2dbd3820d045  inference.py
+d24d50148011e72c714df747d0f840b9b25aa1f57d68f6e14dfffaa8841a0b0e  provenance.json
+f9ddbd042be8ebee5660db7877605d867963a6c3b66af38e5debc41d3bd69498  requirements.txt
+baec30ea10906f16adb8c18af7a34023002c1746542612b8b41c9f09e1351351  special_tokens_map.json
+5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060  tokenizer.json
+61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2  tokenizer.model
+89fc4e8286c5201fc7fff22d75af3aa186f1f9c617b41a62a2113b505b8b0561  tokenizer_config.json
+84459976bbc04ef39db5ae44f2fbc8e9a10e204870b0e6bd731c4a088cd522d5  training_config.json

USE_POLICY.md

@@ -0,0 +1,71 @@
+Source: https://ai.google.dev/gemma/prohibited_use_policy
+Retrieved: 2026-09-16
+
+Google reserves the right to update this Gemma Prohibited Use Policy from time
+to time.
+Last modified: February 21, 2024
+You
+may not
+use nor allow others to use Gemma or Model Derivatives to:
+Generate any content, including the outputs or results generated by Gemma or
+Model Derivatives, that infringes, misappropriates, or otherwise violates any
+individual's or entity's rights (including, but not limited to rights in
+copyrighted content).
+Perform or facilitate dangerous, illegal, or malicious activities, including:
+Facilitation or promotion of illegal activities or violations of law,
+such as:
+Promoting or generating content related to child sexual abuse or
+exploitation;
+Promoting or facilitating sale of, or providing instructions for
+synthesizing or accessing, illegal substances, goods, or services;
+Facilitating or encouraging users to commit any type of crimes; or
+Promoting or generating violent extremism or terrorist content.
+Engagement in the illegal or unlicensed practice of any vocation or
+profession including, but not limited to, legal, medical, accounting, or
+financial professional practices.
+Abuse, harm, interference, or disruption of services (or enable others to
+do the same), such as:
+Promoting or facilitating the generation or distribution of spam; or
+Generating content for deceptive or fraudulent activities, scams,
+phishing, or malware.
+Attempts to override or circumvent safety filters or intentionally drive
+Gemma or Model Derivatives to act in a manner that contravenes this Gemma
+Prohibited Use Policy.
+Generation of content that may harm or promote the harm of individuals or
+a group, such as:
+Generating content that promotes or encourages hatred;
+Facilitating methods of harassment or bullying to intimidate, abuse,
+or insult others;
+Generating content that facilitates, promotes, or incites violence;
+Generating content that facilitates, promotes, or encourages self
+harm;
+Generating personally identifying information for distribution or
+other harms;
+Tracking or monitoring people without their consent;
+Generating content that may have unfair or adverse impacts on people,
+particularly impacts related to sensitive or protected
+characteristics; or
+Generating, gathering, processing, or inferring sensitive personal or
+private information about individuals without obtaining all rights,
+authorizations, and consents required by applicable laws.
+Generate and distribute content intended to misinform, misrepresent or
+mislead, including:
+Misrepresentation of the provenance of generated content by claiming
+content was created by a human, or represent generated content as
+original works, in order to deceive;
+Generation of content that impersonates an individual (living or dead)
+without explicit disclosure, in order to deceive;
+Misleading claims of expertise or capability made particularly in
+sensitive areas (e.g. health, finance, government services, or legal);
+Making automated decisions in domains that affect material or individual
+rights or well-being (e.g., finance, legal, employment, healthcare,
+housing, insurance, and social welfare);
+Generation of defamatory content, including defamatory statements,
+images, or audio content; or
+Engaging in the unauthorized or unlicensed practice of any profession
+including, but not limited to, financial, legal, medical/health, or
+related professional practices.
+Generate sexually explicit content, including content created for the
+purposes of pornography or sexual gratification (e.g. sexual chatbots). Note
+that this does not include content created for scientific, educational,
+documentary, or artistic purposes.

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "down_proj",
     "q_proj",
-    "up_proj",
     "v_proj",
+    "gate_proj",
     "o_proj",
-    "down_proj",
     "k_proj",
-    "gate_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dd88dab19511aa4362cc831a9c977d5116a9878aaa50471616700c838b4d68a
+oid sha256:6ba739a50256d7a882c1f75cd66075a4e1da9d6f40445edbb5c6b63af2e973c8
 size 108113968

all_results.json

@@ -1,20 +0,0 @@
-{
-    "epoch": 0.99996003996004,
-    "eval_logits/chosen": -12.166614532470703,
-    "eval_logits/rejected": -12.147299766540527,
-    "eval_logps/chosen": -142.026123046875,
-    "eval_logps/rejected": -180.4429931640625,
-    "eval_loss": 0.34084001183509827,
-    "eval_rewards/accuracies": 0.8528274893760681,
-    "eval_rewards/chosen": -7.012221336364746,
-    "eval_rewards/margins": 3.4816415309906006,
-    "eval_rewards/rejected": -10.49386215209961,
-    "eval_runtime": 5774.0601,
-    "eval_samples_per_second": 1.926,
-    "eval_steps_per_second": 1.926,
-    "total_flos": 6.814329228177064e+18,
-    "train_loss": 0.41961536931869625,
-    "train_runtime": 114854.9374,
-    "train_samples_per_second": 0.872,
-    "train_steps_per_second": 0.109
-}

chat_template.jinja

@@ -0,0 +1,5 @@
+{{ '<bos>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<start_of_turn>user
+' + content + '<end_of_turn>
+<start_of_turn>model
+' }}{% elif message['role'] == 'assistant' %}{{ content + '<end_of_turn>
+' }}{% endif %}{% endfor %}

eval_results.json

@@ -1,15 +0,0 @@
-{
-    "epoch": 0.99996003996004,
-    "eval_logits/chosen": -12.166614532470703,
-    "eval_logits/rejected": -12.147299766540527,
-    "eval_logps/chosen": -142.026123046875,
-    "eval_logps/rejected": -180.4429931640625,
-    "eval_loss": 0.34084001183509827,
-    "eval_rewards/accuracies": 0.8528274893760681,
-    "eval_rewards/chosen": -7.012221336364746,
-    "eval_rewards/margins": 3.4816415309906006,
-    "eval_rewards/rejected": -10.49386215209961,
-    "eval_runtime": 5774.0601,
-    "eval_samples_per_second": 1.926,
-    "eval_steps_per_second": 1.926
-}

inference.py

@@ -0,0 +1,68 @@
+import torch
+from peft import PeftModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+ADAPTER_ID = 'mario-rc/emotional-rlaif-dpo-gemma-2-9b-it'
+BASE_ID = 'google/gemma-2-9b-it'
+# Pin the base revision checked when this release was prepared.
+BASE_REVISION = '11c9b309abf73637e4b6f9a3fa1e92e615547819'
+# Set this to a commit hash from this adapter's Files and versions tab for a pinned run.
+ADAPTER_REVISION = "main"
+
+SYSTEM_PROMPT = """You are an expert at creating dialogues.
+
+Dialogue and emotional structure:
+Human: (SADNESS) PROMPT.
+Chatbot: (SADNESS) RESPONSE_1. (HAPPINESS) RESPONSE_2. (NEUTRAL) RESPONSE_3.
+
+Dialogue rules:
+The response must be open-domain curated. The response should be coherent, empathetic, engaging and proactive.
+The chatbot RESPONSE is composed of 3 different sentences (RESPONSE_1, RESPONSE_2 and RESPONSE_3), separated by a period.
+Between RESPONSE_1, RESPONSE_2 and RESPONSE_3 should be a max length of 20-25 words.
+RESPONSE_3 must be open-ended to follow-up the conversation, so the Human is encouraged to answer with a full long sentence. Avoid yes/no questions.
+
+Emotional response rules:
+RESPONSE_1 must contain a SADNESS tone.
+RESPONSE_2 must contain a HAPPINESS tone.
+RESPONSE_3 must contain a NEUTRAL tone.
+
+Answer in a single turn to Human. Follow exactly the emotional structure and the emotional and dialogue rules."""
+
+def main():
+    tokenizer = AutoTokenizer.from_pretrained(
+        ADAPTER_ID, revision=ADAPTER_REVISION, trust_remote_code=False,
+    )
+    base = AutoModelForCausalLM.from_pretrained(
+        BASE_ID, revision=BASE_REVISION, trust_remote_code=False,
+        torch_dtype=torch.bfloat16, device_map="auto",
+    )
+    model = PeftModel.from_pretrained(base, ADAPTER_ID, revision=ADAPTER_REVISION)
+    model.eval()
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": "(SADNESS) I feel overwhelmed by my exams."},
+    ]
+    prompt = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True,
+    )
+    inputs = tokenizer(prompt, add_special_tokens=False, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    stop_ids = model.generation_config.eos_token_id
+    stop_ids = list(stop_ids) if isinstance(stop_ids, (list, tuple)) else [stop_ids]
+    stop_ids = sorted({i for i in stop_ids + [tokenizer.eos_token_id] if i is not None})
+    # Include turn-ending tokens, which are not EOS in every base tokenizer.
+    for token in ['<end_of_turn>']:
+        token_id = tokenizer.convert_tokens_to_ids(token)
+        if token_id is not None and token_id != tokenizer.unk_token_id and token_id not in stop_ids:
+            stop_ids.append(token_id)
+    pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else stop_ids[0]
+    with torch.inference_mode():
+        output = model.generate(
+            **inputs, max_new_tokens=128, do_sample=False,
+            eos_token_id=stop_ids, pad_token_id=pad_id,
+        )
+    answer = tokenizer.decode(output[0, inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
+    print(answer)
+
+if __name__ == "__main__":
+    main()

provenance.json

@@ -0,0 +1,18 @@
+{
+  "repository": "mario-rc/emotional-rlaif-dpo-gemma-2-9b-it",
+  "base_model": "google/gemma-2-9b-it",
+  "base_revision_checked": "11c9b309abf73637e4b6f9a3fa1e92e615547819",
+  "base_revision_note": "Revision pinned for this release; historical training revision was not recorded.",
+  "method": "DPO",
+  "source_run": "dpo_bs64_1ep",
+  "source_adapter": "phase3-rlaif-alignment/rlaif-model/rlaif-llama-factory-training/saves/gemma-2-9b-it/lora/dpo_bs64_1ep",
+  "adapter_sha256": "6ba739a50256d7a882c1f75cd66075a4e1da9d6f40445edbb5c6b63af2e973c8",
+  "seed": 42,
+  "sft_initialization": "phase2-sft-alignment/sft-model/sft-llama-factory-training/saves/gemma-2-9b-it/lora/sft_3ep",
+  "sft_adapter_sha256": "104d3f4e5e031d27c2b9e906a3b83f079e663d4c7d4ef4aafb6f917b992859ef",
+  "lineage": "base -> matching SFT -> DPO",
+  "training_config_sha256": "3a03d62cc5302b1c78616237fdf95dd9c879a9e5364bd72fb6523cfd4af82fbd",
+  "release_date": "2026-09-16",
+  "numerical_evaluation_published": false,
+  "tokenizer_release_note": "Vocabulary unchanged; chat_template regenerated from the exact LLaMA-Factory family template so system prompts and generation match training."
+}

requirements.txt

@@ -0,0 +1,5 @@
+torch==2.5.1
+transformers==4.45.2
+peft==0.11.1
+accelerate==0.34.0
+sentencepiece

tokenizer_config.json

@@ -2000,10 +2000,10 @@
     "<end_of_turn>"
   ],
   "bos_token": "<bos>",
-  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template": "{{ '<bos>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<start_of_turn>user\n' + content + '<end_of_turn>\n<start_of_turn>model\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<end_of_turn>\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
-  "model_max_length": 1000000000000000019884624838656,
+  "model_max_length": 2048,
   "pad_token": "<pad>",
   "padding_side": "right",
   "sp_model_kwargs": {},

train_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 0.99996003996004,
-    "total_flos": 6.814329228177064e+18,
-    "train_loss": 0.41961536931869625,
-    "train_runtime": 114854.9374,
-    "train_samples_per_second": 0.872,
-    "train_steps_per_second": 0.109
-}

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c06fe1f1f49ba1d8abd39b6e211d06afdaec414f07cd625cdd9521622cb8f576
-size 5368

training_config.json

@@ -0,0 +1,34 @@
+{
+  "stage": "dpo",
+  "template": "gemma",
+  "cutoff_len": 2048,
+  "dataset": "dpo_preference_dataset",
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 64,
+  "learning_rate": 5e-06,
+  "num_train_epochs": 1.0,
+  "lr_scheduler_type": "cosine",
+  "warmup_ratio": 0.1,
+  "bf16": true,
+  "pref_beta": 0.1,
+  "pref_loss": "sigmoid",
+  "weight_decay": 0.0,
+  "max_grad_norm": 1.0,
+  "seed": 42,
+  "data_seed": null,
+  "max_steps": -1,
+  "optim": "adamw_torch",
+  "lora_rank": 8,
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "sft_initialization": "../../../phase2-sft-alignment/sft-model/sft-llama-factory-training/saves/gemma-2-9b-it/lora/sft_3ep"
+}