Auditing Preference Biases and Fine-Tuning Language Models with Direct Preference Optimization on Anthropic HH-RLHF Using TRL and LoRA
In this tutorial, we design an end-to-end preference-learning workflow utilizing the Anthropic HH-RLHF dataset and Direct Preference Optimization (DPO). We start by making ready a sturdy Colab surroundings, loading and parsing chosen–rejected response pairs, and auditing the dataset for structural and length-based choice biases. We then run lexical shortcut diagnostics to find out whether or not surface-level linguistic patterns can separate most well-liked from rejected responses, put together conversational information with tokenizer-aware size filtering, and assemble a version-robust DPO coaching pipeline with TRL and non-obligatory LoRA adaptation. Finally, we fine-tune a Qwen2.5-0.5B-Instruct mannequin, consider reward accuracy and coaching conduct, analyze efficiency throughout particular person HH-RLHF subsets, examine potential size bias, generate pattern responses, and save the ensuing coverage for additional experimentation.
import dataclasses
import importlib.util
import examine
import os
import re
import subprocess
import sys
import warnings
warnings.filterwarnings("ignore", class=UserWarning)
REQUIRED = ["trl>=0.12", "transformers>=4.45", "accelerate", "datasets", "peft", "scikit-learn"]
def ensure_deps():
"""Install in ONE pip name so the resolver picks a mutually appropriate set."""
strive:
import trl
import transformers
return False
besides ImportError:
print("Installing dependencies...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "-U", *REQUIRED])
return True
def drop_broken_torchao():
"""Colab ships torchao 0.10.0; peft calls for >0.16 and raises reasonably than skipping.
Nothing right here makes use of torchao, so eradicating it's safer than upgrading (an improve can
drag in a torch construct that doesn't match this runtime)."""
if importlib.util.find_spec("torchao") is None:
return False
strive:
from peft.import_utils import is_torchao_available
is_torchao_available()
return False
besides ImportError:
print("Removing incompatible torchao (unused, however peft raises on it)...")
subprocess.name([sys.executable, "-m", "pip", "uninstall", "-y", "-q", "torchao"])
return True
besides Exception:
return False
_installed = ensure_deps()
_removed = drop_broken_torchao() if not _installed else False
if _installed or _removed:
print("nEnvironment modified. RESTART THE RUNTIME (Runtime > Restart session), "
"then run this cell once more.")
elevate SystemExit(0)
import numpy as np
import pandas as pd
import torch
import matplotlib.pyplot as plt
from datasets import load_dataset, concatenate_datasets
from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
from sklearn.feature_extraction.textual content import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report, roc_auc_score
import transformers
import trl
from trl import DPOConfig, DPOTrainer
def patch_peft_torchao():
"""Belt and braces: if torchao survived the uninstall, cease peft elevating on it."""
strive:
from peft import import_utils
from peft.tuners.lora import torchao as lora_torchao
besides ImportError:
return
strive:
import_utils.is_torchao_available()
besides ImportError as exc:
print(f" neutralising peft's torchao examine ({exc})")
import_utils.is_torchao_available = lambda: False
lora_torchao.is_torchao_available = lambda: False
MODEL_ID = "Qwen/Qwen2.5-0.5B-Instruct"
SUBSETS = ["helpful-base", "helpful-rejection-sampled", "helpful-online", "harmless-base"]
N_TRAIN_PER_SUBSET = 120
N_TEST_PER_SUBSET = 30
MAX_LENGTH = 512
MAX_PROMPT_LENGTH = 256
BETA = 0.1
MAX_STEPS = 30
BATCH_SIZE = 1
GRAD_ACCUM = 8
LEARNING_RATE = 5e-6
WARMUP_RATIO = 0.1
LOGGING_STEPS = 5
USE_LORA = True
N_REWARD_EVAL = 40
SEED = 17
OUTPUT_DIR = "/content material/dpo-hh" if os.path.isdir("/content material") else "./dpo-hh"
set_seed(SEED)
rng = np.random.default_rng(SEED)
def report_environment():
from transformers import TrainingArguments
cuda = torch.cuda.is_available()
bf16 = bool(cuda and torch.cuda.is_bf16_supported())
fp16 = bool(cuda and not bf16)
machine = "cuda" if cuda else "cpu"
print(f"python : {sys.model.cut up()[0]}")
print(f"torch : {torch.__version__}")
print(f"transformers : {transformers.__version__}")
print(f"trl : {trl.__version__}")
print(f"Device: {machine} | bf16={bf16} | fp16={fp16}")
if not cuda:
print("CPU fallback is enabled; coaching is deliberately shortened.")
cfg_fields = {f.title for f in dataclasses.fields(DPOConfig)}
trainer_params = set(examine.signature(DPOTrainer.__init__).parameters)
print(f"DPOConfig subclasses TrainingArguments : {issubclass(DPOConfig, TrainingArguments)}")
print(f"DPOConfig fields : {len(cfg_fields)}")
for probe in ("warmup_ratio", "warmup_steps", "beta", "max_length", "max_prompt_length"):
the place = [c for c, s in (("DPOConfig", cfg_fields), ("DPOTrainer", trainer_params))
if probe in s]
print(f" {probe:<20} -> {', '.be a part of(the place) if the place else 'NOT ACCEPTED ANYWHERE'}")
if not issubclass(DPOConfig, TrainingArguments) or "per_device_train_batch_size" not in cfg_fields:
print("n!! DPOConfig seems to be damaged. Reinstall in a single command, then restart:")
print(" pip set up -U trl transformers speed up datasets peft")
return machine, bf16, fp16, cfg_fields, trainer_params
DEVICE, BF16, FP16, CFG_FIELDS, TRAINER_PARAMS = report_environment()
We arrange the required libraries, deal with dependency compatibility points, and configure the primary parameters used all through the tutorial. We additionally initialize reproducibility settings and examine the accessible {hardware}, precision modes, and put in TRL interfaces. This offers us a steady surroundings earlier than we course of the HH-RLHF dataset and prepare the choice mannequin.
def sample_split(ds, n, seed):
return ds.shuffle(seed=seed).choose(vary(min(n, len(ds)))).flatten_indices()
def load_hh():
train_parts, test_parts = [], []
for i, subset in enumerate(SUBSETS):
ds = load_dataset("Anthropic/hh-rlhf", data_dir=subset)
tr = sample_split(ds["train"], N_TRAIN_PER_SUBSET, SEED + i)
te = sample_split(ds["test"], N_TEST_PER_SUBSET, SEED + i)
train_parts.append(tr.add_column("supply", [subset] * len(tr)))
test_parts.append(te.add_column("supply", [subset] * len(te)))
return concatenate_datasets(train_parts), concatenate_datasets(test_parts)
raw_train, raw_test = load_hh()
print(f"nRaw sampled rows -> prepare={len(raw_train)}, check={len(raw_test)}")
print(pd.Series(raw_train["source"]).value_counts().sort_index().to_string())
TURN_RE = re.compile(r"nn(Human|Assistant):[ ]?")
def parse_transcript(textual content):
if not isinstance(textual content, str) or not textual content.strip():
return None
elements = TURN_RE.cut up(textual content)
if elements[0].strip():
return None
roles, contents = elements[1::2], elements[2::2]
if len(roles) != len(contents) or len(roles) < 2:
return None
msgs = [{"role": "user" if r == "Human" else "assistant", "content": c.strip()}
for r, c in zip(roles, contents)]
if msgs[0]["role"] != "person" or msgs[-1]["role"] != "assistant":
return None
if any(a["role"] == b["role"] for a, b in zip(msgs, msgs[1:])):
return None
if any(not m["content"] for m in msgs):
return None
return msgs
def to_pair(row):
ch = parse_transcript(row["chosen"])
rj = parse_transcript(row["rejected"])
okay = ch isn't None and rj isn't None and ch[:-1] == rj[:-1]
return {
"okay": bool(okay),
"immediate": ch[:-1] if okay else [],
"chosen": [ch[-1]] if okay else [],
"rejected": [rj[-1]] if okay else [],
"prompt_turns": len(ch) - 1 if okay else 0,
"supply": row["source"],
}
parsed_train = raw_train.map(to_pair, remove_columns=raw_train.column_names).filter(lambda r: r["ok"])
parsed_test = raw_test.map(to_pair, remove_columns=raw_test.column_names).filter(lambda r: r["ok"])
print(f"nValid parsed rows -> prepare={len(parsed_train)}, check={len(parsed_test)}")
similar = sum(1 for c, r in zip(parsed_train["chosen"], parsed_train["rejected"])
if c[0]["content"] == r[0]["content"])
print(f"Identical completion pairs in sampled prepare: {similar}")
We load samples from the totally different Anthropic HH-RLHF subsets and create balanced coaching and testing datasets. We parse every dialog into structured person and assistant messages whereas guaranteeing that chosen and rejected responses share the identical conversational prefix. We then filter invalid pairs in order that we work solely with correctly aligned choice examples.
audit = pd.DataFrame({
"supply": parsed_train["source"],
"prompt_turns": parsed_train["prompt_turns"],
"chosen_words": [len(c[0]["content"].cut up()) for c in parsed_train["chosen"]],
"rejected_words": [len(r[0]["content"].cut up()) for r in parsed_train["rejected"]],
})
audit["length_delta"] = audit["chosen_words"] - audit["rejected_words"]
abstract = audit.groupby("supply").agg(
pairs=("chosen_words", "dimension"),
chosen_words_mean=("chosen_words", "imply"),
rejected_words_mean=("rejected_words", "imply"),
median_turns=("prompt_turns", "median"),
mean_length_delta=("length_delta", "imply"),
).spherical(2)
print("nPreference-pair audit:")
print(abstract.to_string())
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
abstract["mean_length_delta"].plot(variety="barh", ax=axes[0], colour="#4c72b0")
axes[0].axvline(0, colour="0.3", lw=1)
axes[0].set_title("imply(chosen − rejected) phrases")
axes[0].set_ylabel("")
for src, grp in audit.groupby("supply"):
axes[1].hist(grp["length_delta"], bins=30, histtype="step", lw=1.6, label=src)
axes[1].axvline(0, colour="0.3", lw=1)
axes[1].set_title("per-pair size delta")
axes[1].legend(fontsize=7)
plt.tight_layout()
plt.present()
print("nSanitized structural preview (person textual content isn't printed):")
for i in vary(min(3, len(audit))):
r = audit.iloc[i]
print({"supply": r["source"], "prompt_turns": int(r["prompt_turns"]),
"chosen_words": int(r["chosen_words"]), "rejected_words": int(r["rejected_words"])})
def build_lexical_dataset(ds):
chosen_txt = [c[0]["content"] for c in ds["chosen"]]
rejected_txt = [r[0]["content"] for r in ds["rejected"]]
texts = chosen_txt + rejected_txt
labels = np.concatenate([np.ones(len(chosen_txt), int), np.zeros(len(rejected_txt), int)])
pair_id = np.concatenate([np.arange(len(chosen_txt)), np.arange(len(rejected_txt))])
assert texts[: len(chosen_txt)] == chosen_txt and labels[: len(chosen_txt)].all()
assert not labels[len(chosen_txt):].any()
return np.array(texts, dtype=object), labels, pair_id
def run_lexical_diagnostic(texts, labels, pair_id, tag="noticed"):
pairs = np.distinctive(pair_id)
shuffled = rng.permutation(pairs)
test_pairs = set(shuffled[: len(shuffled) // 2].tolist())
is_test = np.array([p in test_pairs for p in pair_id])
vec = TfidfVectorizer(ngram_range=(1, 2), min_df=2, max_features=20000, sublinear_tf=True)
Xtr = vec.fit_transform(texts[~is_test])
Xte = vec.remodel(texts[is_test])
clf = LogisticRegression(max_iter=2000).match(Xtr, labels[~is_test])
pred = clf.predict(Xte)
prob = clf.predict_proba(Xte)[:, 1]
acc = accuracy_score(labels[is_test], pred)
auc = roc_auc_score(labels[is_test], prob)
print(f"Lexical diagnostic ({tag}) accuracy: {acc:.3f}")
print(f"Lexical diagnostic ({tag}) ROC-AUC: {auc:.3f}")
return acc, auc, clf, labels[is_test], pred
print("nTraining a lexical diagnostic to detect straightforward choice shortcuts...")
texts, labels, pair_id = build_lexical_dataset(parsed_train)
acc, auc, clf, y_true, y_pred = run_lexical_diagnostic(texts, labels, pair_id)
print(classification_report(y_true, y_pred, target_names=["rejected", "chosen"], digits=3))
perm = rng.permutation(len(labels))
_, auc_perm, _, _, _ = run_lexical_diagnostic(texts, labels[perm], pair_id, tag="permuted labels")
print(f"Chance baseline from permuted labels: AUC {auc_perm:.3f}")
if abs(auc - 0.5) <= abs(auc_perm - 0.5) + 0.02:
print("-> noticed AUC is inside permutation noise: no detectable lexical shortcut.")
elif auc < 0.5:
print("-> noticed AUC is BELOW likelihood past noise: examine label ordering upstream.")
else:
print("-> noticed AUC is ABOVE likelihood: an actual lexical shortcut exists on this pattern.")
coefs = np.type(np.abs(clf.coef_.ravel()))[-20:]
print(f"Top-20 absolute lexical coefficient vary: {coefs[0]:.3f} to {coefs[-1]:.3f}")
print("Feature strings are deliberately not printed as a result of the supply corpus might comprise offensive textual content.")
We analyze the choice pairs to measure variations in response size, dialog depth, and source-specific conduct. We additionally prepare a TF-IDF and logistic regression diagnostic to check whether or not easy lexical patterns can distinguish chosen responses from rejected ones. This helps us detect shortcuts that the language mannequin might probably exploit as a substitute of studying the supposed choice sign.
print("nPreparing conversational DPO information...")
tok = AutoTokenizer.from_pretrained(MODEL_ID)
if tok.pad_token is None:
tok.pad_token = tok.eos_token
CHATML = (
"{% for m in messages %}"
"{im_start}"
"{% endfor %}"
"{% if add_generation_prompt %}{im_start}{% endif %}"
)
if getattr(tok, "chat_template", None) is None:
tok.chat_template = CHATML
print("Tokenizer had no chat template; put in a ChatML fallback.")
def add_lengths(row):
prompt_txt = tok.apply_chat_template(row["prompt"], tokenize=False, add_generation_prompt=True)
n_prompt = len(tok(prompt_txt, add_special_tokens=False)["input_ids"])
n_ch = len(tok(row["chosen"][0]["content"], add_special_tokens=False)["input_ids"])
n_rj = len(tok(row["rejected"][0]["content"], add_special_tokens=False)["input_ids"])
return {"n_prompt": n_prompt, "n_total": n_prompt + max(n_ch, n_rj)}
def suits(row):
return row["n_prompt"] <= MAX_PROMPT_LENGTH and row["n_total"] <= MAX_LENGTH
dpo_train_full = parsed_train.map(add_lengths).filter(suits)
dpo_test_full = parsed_test.map(add_lengths).filter(suits)
test_sources = record(dpo_test_full["source"])
test_prompts = record(dpo_test_full["prompt"])
test_chosen = record(dpo_test_full["chosen"])
test_rejected = record(dpo_test_full["rejected"])
DPO_COLS = ["prompt", "chosen", "rejected"]
dpo_train = dpo_train_full.remove_columns([c for c in dpo_train_full.column_names if c not in DPO_COLS])
dpo_test = dpo_test_full.remove_columns([c for c in dpo_test_full.column_names if c not in DPO_COLS])
print(f"DPO-ready rows after {MAX_LENGTH}-token filter -> prepare={len(dpo_train)}, check={len(dpo_test)}")
print("DPO schema:", dict(dpo_train.options))
def split_kwargs(needed, legitimate):
return ({okay: v for okay, v in needed.objects() if okay in legitimate},
{okay: v for okay, v in needed.objects() if okay not in legitimate})
def build_dpo_config(needed):
saved, dropped = split_kwargs(needed, CFG_FIELDS)
if "warmup_ratio" in dropped and "warmup_steps" in CFG_FIELDS:
steps = max(1, int(dropped.pop("warmup_ratio") * needed.get("max_steps", 100)))
saved["warmup_steps"] = steps
print(f" warmup_ratio unsupported right here -> transformed to warmup_steps={steps}")
forwarded, truly_dropped = split_kwargs(dropped, TRAINER_PARAMS)
if forwarded:
print(" forwarded to DPOTrainer:", sorted(forwarded))
if truly_dropped:
print(" dropped (accepted nowhere on this construct):", sorted(truly_dropped))
if "max_prompt_length" in truly_dropped:
print(" -> innocent: the token filter in part 7 already caps prompts")
return DPOConfig(**saved), forwarded
wanted_args = dict(
output_dir=OUTPUT_DIR,
max_steps=MAX_STEPS,
per_device_train_batch_size=BATCH_SIZE,
per_device_eval_batch_size=BATCH_SIZE,
gradient_accumulation_steps=GRAD_ACCUM,
learning_rate=LEARNING_RATE,
warmup_ratio=WARMUP_RATIO,
logging_steps=LOGGING_STEPS,
save_strategy="no",
report_to=[],
remove_unused_columns=False,
bf16=BF16,
fp16=FP16,
seed=SEED,
beta=BETA,
max_length=MAX_LENGTH,
max_prompt_length=MAX_PROMPT_LENGTH,
)
print("nBuilding DPOConfig for the put in TRL...")
args, forwarded_to_trainer = build_dpo_config(wanted_args)
print(" DPOConfig constructed OK")
def build_model():
dtype = torch.bfloat16 if BF16 else torch.float32
strive:
mannequin = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=dtype)
besides TypeError:
mannequin = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=dtype)
mannequin.config.use_cache = False
return mannequin
peft_config = None
if USE_LORA:
strive:
from peft import LoraConfig
peft_config = LoraConfig(
r=16, lora_alpha=32, lora_dropout=0.05, bias="none",
task_type="CAUSAL_LM",
)
print(" LoRA enabled (the frozen base doubles because the reference mannequin)")
besides ImportError:
print(" peft not put in -> full fine-tune with an specific reference mannequin")
def build_trainer(mannequin, args, train_ds, eval_ds, tokenizer, peft_config, further):
kwargs = dict(mannequin=mannequin, args=args, train_dataset=train_ds, eval_dataset=eval_ds)
if "processing_class" in TRAINER_PARAMS:
kwargs["processing_class"] = tokenizer
elif "tokenizer" in TRAINER_PARAMS:
kwargs["tokenizer"] = tokenizer
if peft_config isn't None and "peft_config" in TRAINER_PARAMS:
kwargs["peft_config"] = peft_config
elif peft_config is None and "ref_model" in TRAINER_PARAMS:
kwargs["ref_model"] = None
kwargs.replace(further)
print(" DPOTrainer kwargs:", sorted(kwargs))
return DPOTrainer(**kwargs)
print("nBuilding DPOTrainer...")
patch_peft_torchao()
mannequin = build_model()
coach = build_trainer(mannequin, args, dpo_train, dpo_test, tok, peft_config, forwarded_to_trainer)
print(" DPOTrainer constructed OK")
We put together the tokenizer, apply the conversational chat template, and calculate token lengths for each choice pair. We filter examples that exceed our immediate or whole sequence limits and dynamically assemble DPO configuration arguments primarily based on the put in TRL model. We then load the bottom mannequin, configure LoRA when accessible, and construct the DPO coach that we use for fine-tuning.
print(f"nTraining for {MAX_STEPS} steps on {DEVICE} "
f"(efficient batch {BATCH_SIZE * GRAD_ACCUM})...")
train_result = coach.prepare()
print("nTraining metrics:")
for okay, v in sorted(train_result.metrics.objects()):
print(f" {okay:<28} {v}")
print("nEvaluating on held-out pairs...")
eval_metrics = coach.consider()
for okay, v in sorted(eval_metrics.objects()):
if any(t in okay for t in ("accuracies", "margins", "rewards", "loss")):
print(f" {okay:<34} {v:.4f}" if isinstance(v, float) else f" {okay:<34} {v}")
log_df = pd.DataFrame(coach.state.log_history)
if "loss" in log_df.columns:
fig, ax = plt.subplots(figsize=(7, 3.5))
d = log_df.dropna(subset=["loss"])
ax.plot(d["step"], d["loss"], marker="o", ms=3, label="prepare loss")
acc_col = subsequent((c for c in log_df.columns if c.endswith("rewards/accuracies")), None)
if acc_col:
d2 = log_df.dropna(subset=[acc_col])
ax.plot(d2["step"], d2[acc_col], marker="s", ms=3, label="reward accuracy")
ax.axhline(0.5, colour="0.6", lw=0.8, ls="--")
ax.set_xlabel("step")
ax.legend(fontsize=8)
ax.set_title("DPO coaching")
plt.tight_layout()
plt.present()
We prepare the mannequin utilizing Direct Preference Optimization with the configured batch dimension, gradient accumulation, studying price, and optimization steps. We consider the ensuing coverage on held-out choice pairs and examine metrics reminiscent of loss, reward margins, and reward accuracy. We additionally visualize the coaching historical past to look at how preference-learning efficiency modifications all through optimization.
@torch.no_grad()
def completion_logprob(coverage, messages_prompt, message_completion, use_ref=False):
prompt_txt = tok.apply_chat_template(messages_prompt, tokenize=False, add_generation_prompt=True)
full_txt = prompt_txt + message_completion["content"] + tok.eos_token
p_ids = tok(prompt_txt, add_special_tokens=False, return_tensors="pt")["input_ids"]
f_ids = tok(full_txt, add_special_tokens=False, return_tensors="pt",
truncation=True, max_length=MAX_LENGTH)["input_ids"].to(coverage.machine)
begin = min(p_ids.form[1], f_ids.form[1] - 1)
ctx = coverage.disable_adapter() if (use_ref and hasattr(coverage, "disable_adapter")) else None
if ctx isn't None:
with ctx:
logits = coverage(f_ids).logits
else:
logits = coverage(f_ids).logits
logprobs = torch.log_softmax(logits[:, :-1].float(), dim=-1)
targets = f_ids[:, 1:]
picked = logprobs.collect(-1, targets.unsqueeze(-1)).squeeze(-1)
return picked[:, start:].sum().merchandise()
def per_source_reward_accuracy(n=N_REWARD_EVAL):
coverage = coach.mannequin
coverage.eval()
if not hasattr(coverage, "disable_adapter") and getattr(coach, "ref_model", None) is None:
print(" no reference mannequin reachable; skipping per-source evaluation")
return None
idx = rng.permutation(len(test_sources))[:min(n, len(test_sources))]
rows = []
for i in idx:
i = int(i)
rc = completion_logprob(coverage, test_prompts[i], test_chosen[i][0])
rr = completion_logprob(coverage, test_prompts[i], test_rejected[i][0])
refc = completion_logprob(coverage, test_prompts[i], test_chosen[i][0], use_ref=True)
refr = completion_logprob(coverage, test_prompts[i], test_rejected[i][0], use_ref=True)
rows.append({
"supply": test_sources[i],
"margin": BETA * ((rc - refc) - (rr - refr)),
"right": BETA * ((rc - refc) - (rr - refr)) > 0,
"len_delta": len(test_chosen[i][0]["content"].cut up())
- len(test_rejected[i][0]["content"].cut up()),
})
df = pd.DataFrame(rows)
out = df.groupby("supply").agg(
n=("right", "dimension"),
reward_accuracy=("right", "imply"),
mean_margin=("margin", "imply"),
mean_len_delta=("len_delta", "imply"),
).spherical(3)
print(out.to_string())
longer_wins = (df["correct"] == (df["len_delta"] > 0)).imply()
print(f"n settlement between 'mannequin prefers chosen' and 'chosen is longer': {longer_wins:.3f}")
print(" (close to 0.5 = no size shortcut; close to 1.0 = the coverage is usually rating by size)")
return out
print(f"nPer-source reward accuracy on {N_REWARD_EVAL} held-out pairs:")
strive:
per_source = per_source_reward_accuracy()
besides Exception as exc:
print(f" skipped: {sort(exc).__name__}: {exc}")
per_source = None
def generate(messages, max_new_tokens=96):
textual content = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
enc = tok(textual content, return_tensors="pt").to(coach.mannequin.machine)
with torch.no_grad():
out = coach.mannequin.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False,
pad_token_id=tok.pad_token_id)
return tok.decode(out[0][enc["input_ids"].form[1]:], skip_special_tokens=True).strip()
probes = [
[{"role": "user", "content": "My laptop fan is suddenly very loud. What should I check first?"}],
[{"role": "user", "content": "Explain in two sentences why DPO does not need a separate reward model."}],
]
print("nSample generations from the tuned coverage:")
for p in probes:
print(f"n person : {p[0]['content']}")
print(f" assistant : {generate(p)}")
coach.save_model(OUTPUT_DIR)
tok.save_pretrained(OUTPUT_DIR)
print(f"nSaved to {OUTPUT_DIR}")
print("""
Reading the outcomes
* At MAX_STEPS=30 on CPU this can be a smoke check, not a educated mannequin. Reward accuracy
close to 0.5 is the anticipated end result; elevate MAX_STEPS on a GPU earlier than concluding something.
* The quantity to observe is the per-source desk, not the mixture. If harmless-base
reward accuracy drops whereas the useful subsets rise, the coverage is studying the
size asymmetry seen within the part 5 audit reasonably than the supposed choice.
* If a TRL name fails after an improve, the surroundings report on the prime names which
class accepts which argument in your construct; lengthen `wanted_args` from that record.
""")
We calculate per-source reward accuracy and evaluate coverage and reference-model log chances to look at whether or not the mannequin genuinely prefers the chosen responses. We examine the connection between choice choices and response-length variations, then generate pattern solutions from the tuned coverage to examine its conduct qualitatively. Finally, we save each the educated mannequin and tokenizer in order that we will reuse the ensuing DPO coverage in later experiments.
In conclusion, we developed an entire DPO-based preference-learning pipeline that goes past merely fine-tuning a language mannequin on chosen and rejected responses. We examined the HH-RLHF information for size asymmetries and lexical shortcuts, enforced constant conversational formatting and token limits, and used a versatile coaching setup that adapts to variations throughout TRL and Transformers variations. We additionally evaluated the tuned coverage at each the mixture and per-source ranges, permitting us to establish whether or not enhancements mirror real choice studying or undesirable shortcuts reminiscent of favoring longer solutions. By combining dataset auditing, diagnostic evaluation, environment friendly LoRA-based DPO coaching, reward analysis, and technology testing, we established a framework for learning and bettering choice alignment in language fashions.
Check out the FULL CODES here. Also, be at liberty to observe us on Twitter and don’t overlook to hitch our 150k+ML SubReddit and Subscribe to our Newsletter. Wait! are you on telegram? now you can join us on telegram as well.
Need to accomplice with us for selling your GitHub Repo OR Hugging Face Page OR Product Release OR Webinar and many others.? Connect with us
The publish Auditing Preference Biases and Fine-Tuning Language Models with Direct Preference Optimization on Anthropic HH-RLHF Using TRL and LoRA appeared first on MarkTechPost.
