From b8b52163674c4f8ece43b840778ac2e6c8f37f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E7=AB=8B=E5=AE=8F?= <曾立宏@buaa.edu.cn> Date: Wed, 19 Aug 2026 21:38:23 +0800 Subject: [PATCH 1/4] feat(sft): baseline evaluator (choice metrics + e2e), 7B LoRA baseline launcher, verl LoRA->HF merge tool (phase 13) --- script/verl/sft/evaluate_baseline.py | 250 +++++++++++++++++++++++ script/verl/sft/merge_lora_checkpoint.py | 95 +++++++++ script/verl/sft/run_baseline.sh | 176 ++++++++++++++++ 3 files changed, 521 insertions(+) create mode 100644 script/verl/sft/evaluate_baseline.py create mode 100644 script/verl/sft/merge_lora_checkpoint.py create mode 100644 script/verl/sft/run_baseline.sh diff --git a/script/verl/sft/evaluate_baseline.py b/script/verl/sft/evaluate_baseline.py new file mode 100644 index 0000000..95b8fea --- /dev/null +++ b/script/verl/sft/evaluate_baseline.py @@ -0,0 +1,250 @@ +"""SFT baseline evaluator: generate with a model and score choice-protocol output. + +Runs a model (base HF model or a LoRA-merged HF directory) over a Phase-8 SFT +parquet split and reports the two-stage choice-protocol metrics using the SHARED +choice-aware evaluation layer (``agent.evaluation.classification``), so the +metrics exactly match the frozen parser / contract (no re-implementation). + +For each row the prompt is the exported conversation WITHOUT the assistant gold +(loss target is never shown at inference). Generation is fixed and identical +for every model: greedy, ``do_sample=False``, ``num_beams=1``, fixed +``max_new_tokens`` and seed. + +Metrics (per the Phase-13 spec): +- stage1: format_valid, contract_valid, recall@5 (GT in the 5 predicted ids) +- stage2: format_valid, contract_valid, accuracy-overall, + accuracy-when-GT-in-candidates (and counts) +- end-to-end: stage1 recall (same source) AND stage2 correct, per source_id + (stage1 recall x stage2 final correctness) + +Usage (server, in the SFT venv): + python -m script.verl.sft.evaluate_baseline \ + --model-path \ + --data data/sft/pers_info/test.parquet \ + --registry cfg/task/registry/pers_info.registry.json \ + --max-new-tokens 128 --seed 42 --report +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Iterable +import sys + +from agent.evaluation.classification import ( + evaluate_stage1_choices, + evaluate_stage2_choices, +) +from agent.task.contracts import LeafRegistry +from agent.task.prompt_choices import PromptChoiceRegistry + +FULL_CATALOG_REGISTRY = "pers_info" # evaluator is registry-driven, not hard-coded + + +def aggregate_baseline(records: list[dict[str, Any]]) -> dict[str, Any]: + """Aggregate per-row evaluation records into the Phase-13 metric table. + + ``records`` items: {stage, source_id, ground_truth, candidates, + format_valid, contract_valid, correct, recalled} (choice-decode already + applied by the shared evaluator). Pure function, no transformers/torch. + """ + by_stage: dict[str, list[dict]] = defaultdict(list) + for record in records: + by_stage[record["stage"]].append(record) + + def summarize(stage_rows: Iterable[dict]) -> dict[str, Any]: + rows = list(stage_rows) + n = len(rows) + format_valid = sum(r["format_valid"] for r in rows) / n if n else 0.0 + contract_valid = sum(r["contract_valid"] for r in rows) / n if n else 0.0 + return { + "n": n, + "format_valid": format_valid, + "contract_valid": contract_valid, + } + + # --- stage 1 --- + s1 = dict(summarize(by_stage["stage1"])) + n_s1 = s1["n"] + s1["recall_at_5"] = sum(r["recalled"] for r in by_stage["stage1"]) / n_s1 if n_s1 else 0.0 + s1["recalled_count"] = sum(r["recalled"] for r in by_stage["stage1"]) + + # --- stage 2 --- + s2 = dict(summarize(by_stage["stage2"])) + n_s2 = s2["n"] + correct = sum(r["correct"] for r in by_stage["stage2"]) + s2["accuracy_overall"] = correct / n_s2 if n_s2 else 0.0 + gt_in = [r for r in by_stage["stage2"] if r["gt_in_candidates"]] + s2["n_gt_in_candidates"] = len(gt_in) + s2["accuracy_when_gt_in_candidates"] = ( + sum(r["correct"] for r in gt_in) / len(gt_in) if gt_in else 0.0 + ) + s2["correct_count"] = correct + + # --- end-to-end: stage1 recall AND stage2 correct on the same source --- + by_source: dict[str, dict[str, bool]] = defaultdict(dict) + for r in records: + by_source[r["source_id"]][r["stage"]] = r + e2e_correct = sum( + 1 + for per_source in by_source.values() + if per_source.get("stage1", {}).get("recalled") + and per_source.get("stage2", {}).get("correct") + ) + e2e = { + "pairs": len(by_source), + "correct": e2e_correct, + "correct_rate": e2e_correct / len(by_source) if by_source else 0.0, + } + + return {"stage1": s1, "stage2": s2, "end_to_end": e2e} + + +def _evaluate_rows( + rows: list[dict[str, Any]], + registry: LeafRegistry, +) -> list[dict[str, Any]]: + choices = PromptChoiceRegistry.from_registry(registry) + records: list[dict[str, Any]] = [] + for row in rows: + stage = row["stage"] + gt = row["ground_truth"] + source_id = row["source_id"] + if stage == "stage1": + evaluation = evaluate_stage1_choices( + row["completion"], ground_truth=gt, registry=registry, choices=choices + ) + records.append( + { + "stage": "stage1", + "source_id": source_id, + "ground_truth": gt, + "candidates": None, + "format_valid": evaluation.format_valid, + "contract_valid": evaluation.contract_valid, + "recalled": evaluation.ground_truth_recalled, + "correct": False, + "gt_in_candidates": False, + "prediction": evaluation.prediction, + } + ) + else: + candidates = list(row["candidates"]) + evaluation = evaluate_stage2_choices( + row["completion"], ground_truth=gt, candidates=candidates, registry=registry + ) + records.append( + { + "stage": "stage2", + "source_id": source_id, + "ground_truth": gt, + "candidates": candidates, + "format_valid": evaluation.format_valid, + "contract_valid": evaluation.contract_valid, + "correct": evaluation.correct, + "gt_in_candidates": gt in candidates, + "recalled": False, + "prediction": evaluation.prediction, + } + ) + return records + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-path", required=True, help="HF model dir (base or LoRA-merged)") + parser.add_argument("--data", required=True, help="SFT parquet split (e.g. test.parquet)") + parser.add_argument("--registry", required=True, help="Leaf registry JSON") + parser.add_argument("--max-new-tokens", type=int, default=128) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--report", required=True, help="metrics JSON output path") + parser.add_argument("--completions", help="optional path to precomputed completions JSONL; skips generation") + args = parser.parse_args(argv) + + import pyarrow.parquet as pq + + registry = LeafRegistry.from_path(args.registry) + rows = pq.read_table(args.data).to_pylist() + + generation: dict[str, Any] = { + "model_path": args.model_path, + "do_sample": False, + "num_beams": 1, + "max_new_tokens": args.max_new_tokens, + "seed": args.seed, + } + + if args.completions: + records = [] + # completions JSONL: one {"source_id"/"stage"/"completion"} per eval row + with open(args.completions, encoding="utf-8") as handle: + for line in handle: + if line.strip(): + records.append(json.loads(line)) + # rows are in parquet order -> match completion entries by identity (stage+source_id) + by_key = {(r["stage"], r["source_id"]): r for r in records} + enriched = [] + for row in rows: + key = (row["stage"], row["source_id"]) + if key not in by_key: + print(f"error: missing completion for {key}", file=sys.stderr) + return 2 + enriched.append({**row, "completion": by_key[key]["completion"]}) + rows = enriched + else: + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + torch.manual_seed(args.seed) + tokenizer = AutoTokenizer.from_pretrained(args.model_path) + model = AutoModelForCausalLM.from_pretrained( + args.model_path, torch_dtype=torch.bfloat16, device_map="auto" + ) + if tokenizer.pad_token_id is None: + tokenizer.pad_token_id = tokenizer.eos_token_id + model.eval() + import tqdm + + for index in tqdm.tqdm(range(len(rows)), desc="generate"): + row = rows[index] + messages = row["messages"][:2] # system + user, no assistant gold + text = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + inputs = tokenizer(text, return_tensors="pt").to(model.device) + with torch.inference_mode(): + output = model.generate( + **inputs, + do_sample=False, + num_beams=1, + max_new_tokens=args.max_new_tokens, + pad_token_id=tokenizer.eos_token_id, + ) + input_len = inputs["input_ids"].shape[-1] + rows[index]["completion"] = tokenizer.decode( + output[0][input_len:], skip_special_tokens=True + ).strip() + + records = _evaluate_rows(rows, registry) + metrics = aggregate_baseline(records) + + report = { + "metrics": metrics, + "generation": generation, + "per_row": records, + "registry": str(args.registry), + "data": str(args.data), + } + rendered = json.dumps(report, ensure_ascii=False, indent=2) + path = Path(args.report) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + print(json.dumps(metrics, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/script/verl/sft/merge_lora_checkpoint.py b/script/verl/sft/merge_lora_checkpoint.py new file mode 100644 index 0000000..8bd7b25 --- /dev/null +++ b/script/verl/sft/merge_lora_checkpoint.py @@ -0,0 +1,95 @@ +"""Merge a verl LoRA FSDP checkpoint into a standalone HF model directory. + +verl saves LoRA checkpoints in peft-compatible state-dict layout +(``base_model.model....lora_A.default.weight`` + ``lora_train_meta.json``). +This script rebuilds a PeftModel on the base weights, loads the checkpoint +state dict, merges LoRA into the base and saves a plain HF directory that any +evaluator / downstream RL init can load normally (no verl dependency). + +Verification: reports how many checkpoint keys were consumed; a LoRA merge +with the frozen base weights must consume 100% of keys (base + lora). + +Usage: + python -m script.verl.sft.merge_lora_checkpoint \ + --checkpoint \ + --base-model \ + --output +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +import sys + +import torch + +LORA_A = re.compile(r"(.*)\.lora_A\.default\.weight$") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", required=True, help="verl checkpoint dir (has model_world_size_1_rank_0.pt)") + parser.add_argument("--base-model", required=True, help="HF base model dir (same as trained)") + parser.add_argument("--output", required=True, help="merged HF output dir") + args = parser.parse_args(argv) + + ckpt_dir = Path(args.checkpoint) + model_file = ckpt_dir / "model_world_size_1_rank_0.pt" + meta_file = ckpt_dir / "lora_train_meta.json" + if not model_file.is_file(): + print(f"error: {model_file} not found", file=sys.stderr) + return 2 + + sd = torch.load(model_file, map_location="cpu", weights_only=False) + keys = list(sd.keys()) + targets = sorted( + {m.group(1).rsplit(".", 1)[-1] for k in keys for m in [LORA_A.match(k)] if m} + ) + if not targets: + print("error: no LoRA keys found in checkpoint", file=sys.stderr) + return 2 + + meta = {} + if meta_file.is_file(): + meta = json.loads(meta_file.read_text(encoding="utf-8")) + rank = meta.get("r", 8) + alpha = meta.get("lora_alpha", rank) + + from peft import LoraConfig, PeftModel + from transformers import AutoModelForCausalLM, AutoTokenizer + + print(f"[merge] targets: {targets}") + print(f"[merge] r={rank} alpha={alpha} keys={len(keys)}") + base = AutoModelForCausalLM.from_pretrained(args.base_model, torch_dtype=torch.bfloat16) + config = LoraConfig( + r=rank, + lora_alpha=alpha, + target_modules=targets, + task_type="CAUSAL_LM", + ) + peft = PeftModel(base, config) + missing, unexpected = peft.load_state_dict(sd, strict=False) + unexpected = [k for k in unexpected if not k.startswith("base_model.model.model")] + if unexpected: + print(f"warning: {len(unexpected)} unexpected keys (ignored): {unexpected[:5]}") + used = len(keys) - len(unexpected) + ratio = used / len(keys) + print(f"[merge] consumed {used}/{len(keys)} checkpoint keys ({ratio:.1%})") + if ratio < 0.99: + print("error: checkpoint keys consumed < 99%; aborting", file=sys.stderr) + return 1 + merged = peft.merge_and_unload() + out = Path(args.output) + out.mkdir(parents=True, exist_ok=True) + merged.save_pretrained(out) + tokenizer = AutoTokenizer.from_pretrained(args.base_model) + tokenizer.save_pretrained(out) + print(f"[merge] saved merged model -> {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/script/verl/sft/run_baseline.sh b/script/verl/sft/run_baseline.sh new file mode 100644 index 0000000..5839f88 --- /dev/null +++ b/script/verl/sft/run_baseline.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Reproducible Qwen2.5-7B-Instruct SFT/LoRA baseline launcher (Phase 13). +# +# Trains on a Phase-8 choice-protocol SFT parquet split and records every +# hyper-parameter + the resolved verl config so the baseline is fully +# reproducible (this checkpoint seeds the later GRPO/RLOO/ReMax runs). +# +# Pre-gates: contract validation (soft) + token-budget with the real model +# chat template (hard, only blocks if a row exceeds MAX_LENGTH at +# truncation=error). Every run dumps: +# /hyperparams.json (this launcher's explicit knobs) +# /train_config.json (verl resolved Hydra config dump) +# /checkpoints/... (LoRA FSDP checkpoints, save_freq) +# stderr/stdout -> log (val/loss per trainer.test_freq) +# +# Defaults are the Phase-13 "reasonable default" run. All env-driven. +# +# Run from the repo root in the SFT venv: +# DATASET=pers_info DATA_DIR=... MODEL_PATH=... OUTPUT_DIR=... \ +# bash script/verl/sft/run_baseline.sh > baseline.log 2>&1 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +BASE="${P13_BASE:-$REPO_ROOT/.artifacts/p13}" + +DATASET="${DATASET:-pers_info}" +DATA_DIR="${DATA_DIR:-$BASE/sft/$DATASET}" +TRAIN_FILE="${TRAIN_FILE:-$DATA_DIR/train.parquet}" +VAL_FILE="${VAL_FILE:-$DATA_DIR/val.parquet}" +OUTPUT_DIR="${OUTPUT_DIR:-$BASE/outputs/sft-baseline-$DATASET}" +MODEL_PATH="${MODEL_PATH:-$BASE/models/Qwen2.5-7B-Instruct}" +PYTHON_BIN="${PYTHON_BIN:-python}" + +# ---- hyper-parameters (all recorded in hyperparams.json) ---- +SEED="${SEED:-42}" +LR="${LR:-1e-4}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.01}" +EPOCHS="${EPOCHS:-4}" +TRAIN_BATCH_SIZE="${TRAIN_BATCH_SIZE:-8}" +MICRO_BATCH_SIZE_PER_GPU="${MICRO_BATCH_SIZE_PER_GPU:-2}" +MAX_LENGTH="${MAX_LENGTH:-512}" +MAX_TOKEN_PER_GPU="${MAX_TOKEN_PER_GPU:-$MAX_LENGTH}" +LORA_RANK="${LORA_RANK:-8}" +LORA_ALPHA="${LORA_ALPHA:-16}" +TARGET_MODULES="${TARGET_MODULES:-all-linear}" +ATTENTION_IMPL="${ATTENTION_IMPL:-sdpa}" +GRAD_CKPT="${GRAD_CKPT:-true}" +SAVE_FREQ="${SAVE_FREQ:-10}" +TEST_FREQ="${TEST_FREQ:-5}" +NUM_WORKERS="${NUM_WORKERS:-0}" +MAX_CKPT_KEEP="${MAX_CKPT_KEEP:-null}" + +GPUS="${NUM_GPUS:-1}" +GRAD_ACCUM=$(( TRAIN_BATCH_SIZE / (MICRO_BATCH_SIZE_PER_GPU * GPUS) )) + +export TOKENIZERS_PARALLELISM=false +export HYDRA_FULL_ERROR=1 +export DATACLASSIFY_REGISTRY_DIR="${DATACLASSIFY_REGISTRY_DIR:-cfg/task/registry}" + +cd "$REPO_ROOT" +mkdir -p "$OUTPUT_DIR" +if [[ ! -f "$TRAIN_FILE" || ! -f "$VAL_FILE" ]]; then + echo "error: SFT parquet not found in $DATA_DIR" >&2 + exit 2 +fi +if [[ ! -f "$MODEL_PATH/config.json" ]]; then + echo "error: model not found: $MODEL_PATH" >&2 + exit 2 +fi +for split in train val test; do + if [[ ! -f "$DATA_DIR/$split.parquet" ]]; then + echo "warn: token-budget gate needs $DATA_DIR/$split.parquet (missing $split)" >&2 + fi +done + +# ---- soft gate: contract validation report ---- +if "$PYTHON_BIN" -m script.verl.sft.validate \ + --dataset-dir "$DATA_DIR" \ + --registry "cfg/task/registry/$DATASET.registry.json" \ + --corpus "cfg/task/corpus/$DATASET.corpus.json" \ + --metadata-fields field_name field_description \ + --report "$OUTPUT_DIR/validate.report.json" > "$OUTPUT_DIR/validate.out" 2>&1; then + echo "[baseline] validate: valid" +else + echo "[baseline] validate: FAILED (see $OUTPUT_DIR/validate.out)" >&2 +fi + +# ---- hard gate: token budget ---- +if ! "$PYTHON_BIN" -m script.verl.sft.check_token_budget \ + --dataset-dir "$DATA_DIR" \ + --model "$MODEL_PATH" \ + --max-length "$MAX_LENGTH" \ + --report "$OUTPUT_DIR/token_budget.report.json" > "$OUTPUT_DIR/token_budget.out" 2>&1; then + echo "error: token-budget gate failed (see $OUTPUT_DIR/token_budget.out)" >&2 + exit 2 +fi +echo "[baseline] token budget ok (max_length=$MAX_LENGTH)" + +# ---- record hyper-parameters ---- +"$PYTHON_BIN" - < "$OUTPUT_DIR/hyperparams.json" +import json, subprocess, sys +try: + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip() +except Exception: + commit = "n/a" +params = { + "dataset": "$DATASET", + "model_path": "$MODEL_PATH", + "train_file": "$TRAIN_FILE", + "val_file": "$VAL_FILE", + "seed": int("$SEED"), + "learning_rate": float("$LR"), + "weight_decay": float("$WEIGHT_DECAY"), + "epochs": int("$EPOCHS"), + "train_batch_size": int("$TRAIN_BATCH_SIZE"), + "micro_batch_size_per_gpu": int("$MICRO_BATCH_SIZE_PER_GPU"), + "gpus": int("$GPUS"), + "grad_accumulation_steps": int("$GRAD_ACCUM"), + "max_length": int("$MAX_LENGTH"), + "max_token_len_per_gpu": int("$MAX_TOKEN_PER_GPU"), + "lora_rank": int("$LORA_RANK"), + "lora_alpha": int("$LORA_ALPHA"), + "target_modules": "$TARGET_MODULES", + "precision": "bfloat16", + "engine": "fsdp (strategy=fsdp)", + "optimizer": "AdamW", + "scheduler": "constant (lr_warmup_steps_ratio=0)", + "attention_impl": "$ATTENTION_IMPL", + "gradient_checkpointing": "$GRAD_CKPT", + "save_freq": int("$SAVE_FREQ"), + "test_freq": int("$TEST_FREQ"), + "git_commit": commit, +} +json.dump(params, sys.stdout, ensure_ascii=False, indent=2) +print() +PY +echo "[baseline] hyperparams -> $OUTPUT_DIR/hyperparams.json" + +ARGS=( + "data.train_files=$TRAIN_FILE" "data.val_files=$VAL_FILE" + data.messages_key=messages + "data.train_batch_size=$TRAIN_BATCH_SIZE" + "data.micro_batch_size_per_gpu=$MICRO_BATCH_SIZE_PER_GPU" + "data.max_token_len_per_gpu=$MAX_TOKEN_PER_GPU" + "data.max_length=$MAX_LENGTH" + data.use_dynamic_bsz=false + "data.num_workers=$NUM_WORKERS" + "model.path=$MODEL_PATH" + model.use_remove_padding=false + "model.enable_gradient_checkpointing=$GRAD_CKPT" + "+model.override_config.attn_implementation=$ATTENTION_IMPL" + "model.lora_rank=$LORA_RANK" + "model.lora_alpha=$LORA_ALPHA" + "model.target_modules=$TARGET_MODULES" + engine=fsdp + engine.strategy=fsdp + engine.use_torch_compile=false + engine.dtype=bfloat16 + engine.model_dtype=bfloat16 + "engine.seed=$SEED" + "optim.lr=$LR" + "optim.weight_decay=$WEIGHT_DECAY" + "trainer.project_name=dataclassify-sft-baseline" + "trainer.experiment_name=qwen25-7b-sft-baseline-$DATASET" + "trainer.default_local_dir=$OUTPUT_DIR/checkpoints" + "trainer.seed=$SEED" + 'trainer.logger=["console"]' + "trainer.total_epochs=$EPOCHS" + "trainer.total_training_steps=null" + "trainer.save_freq=$SAVE_FREQ" + "trainer.max_ckpt_to_keep=$MAX_CKPT_KEEP" + "trainer.test_freq=$TEST_FREQ" + trainer.resume_mode=disable +) +NUM_GPUS="$GPUS" PYTHON_BIN="$PYTHON_BIN" bash "$SCRIPT_DIR/run.sh" "${ARGS[@]}" From 85c12644bbc5801eb8006c3cc1bbbd3b859b1229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E7=AB=8B=E5=AE=8F?= <曾立宏@buaa.edu.cn> Date: Wed, 19 Aug 2026 22:23:58 +0800 Subject: [PATCH 2/4] feat(sft): persist per-row completions in baseline eval JSON (phase 13) --- script/verl/sft/evaluate_baseline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/verl/sft/evaluate_baseline.py b/script/verl/sft/evaluate_baseline.py index 95b8fea..62014b9 100644 --- a/script/verl/sft/evaluate_baseline.py +++ b/script/verl/sft/evaluate_baseline.py @@ -123,6 +123,7 @@ def _evaluate_rows( "source_id": source_id, "ground_truth": gt, "candidates": None, + "completion": row["completion"], "format_valid": evaluation.format_valid, "contract_valid": evaluation.contract_valid, "recalled": evaluation.ground_truth_recalled, @@ -142,6 +143,7 @@ def _evaluate_rows( "source_id": source_id, "ground_truth": gt, "candidates": candidates, + "completion": row["completion"], "format_valid": evaluation.format_valid, "contract_valid": evaluation.contract_valid, "correct": evaluation.correct, From 41c24e3feff25186ad9931ac2c5c73ba4a79ea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E7=AB=8B=E5=AE=8F?= <曾立宏@buaa.edu.cn> Date: Thu, 20 Aug 2026 00:00:41 +0800 Subject: [PATCH 3/4] refactor(sft): rename Phase-13 E2E metric to proxy_e2e (factorized/proxy baseline evaluator) --- script/verl/sft/evaluate_baseline.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/script/verl/sft/evaluate_baseline.py b/script/verl/sft/evaluate_baseline.py index 62014b9..a8e3e11 100644 --- a/script/verl/sft/evaluate_baseline.py +++ b/script/verl/sft/evaluate_baseline.py @@ -10,12 +10,14 @@ for every model: greedy, ``do_sample=False``, ``num_beams=1``, fixed ``max_new_tokens`` and seed. -Metrics (per the Phase-13 spec): +Metrics (per the Phase-13 spec; Stage 2 here is scored against the +PRE-BUILT gold-containing bundle from the parquet, so the "end-to-end" +metric below is the FACTORIZED / PROXY e2e, not the true pipeline e2e — +see ``script/verl/sft/evaluate_true_e2e.py`` for the real chained evaluator): - stage1: format_valid, contract_valid, recall@5 (GT in the 5 predicted ids) - stage2: format_valid, contract_valid, accuracy-overall, accuracy-when-GT-in-candidates (and counts) -- end-to-end: stage1 recall (same source) AND stage2 correct, per source_id - (stage1 recall x stage2 final correctness) +- proxy_e2e: stage1 recall (same source) AND stage2 correct (gold bundle) Usage (server, in the SFT venv): python -m script.verl.sft.evaluate_baseline \ @@ -84,7 +86,7 @@ def summarize(stage_rows: Iterable[dict]) -> dict[str, Any]: ) s2["correct_count"] = correct - # --- end-to-end: stage1 recall AND stage2 correct on the same source --- + # --- factorized/proxy e2e: stage1 recall AND stage2 correct (gold bundle) --- by_source: dict[str, dict[str, bool]] = defaultdict(dict) for r in records: by_source[r["source_id"]][r["stage"]] = r @@ -100,7 +102,7 @@ def summarize(stage_rows: Iterable[dict]) -> dict[str, Any]: "correct_rate": e2e_correct / len(by_source) if by_source else 0.0, } - return {"stage1": s1, "stage2": s2, "end_to_end": e2e} + return {"stage1": s1, "stage2": s2, "proxy_e2e": e2e} def _evaluate_rows( From e05f7edd13fc777c71200192fff95efa060cf160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E7=AB=8B=E5=AE=8F?= <曾立宏@buaa.edu.cn> Date: Thu, 20 Aug 2026 00:43:37 +0800 Subject: [PATCH 4/4] fix(sft): baseline launcher review fixes - hard contract-validation gate, bounded checkpoint defaults, grad-accum divisibility guard, RL-init wording --- script/verl/sft/run_baseline.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/script/verl/sft/run_baseline.sh b/script/verl/sft/run_baseline.sh index 5839f88..02dfadc 100644 --- a/script/verl/sft/run_baseline.sh +++ b/script/verl/sft/run_baseline.sh @@ -3,10 +3,13 @@ # # Trains on a Phase-8 choice-protocol SFT parquet split and records every # hyper-parameter + the resolved verl config so the baseline is fully -# reproducible (this checkpoint seeds the later GRPO/RLOO/ReMax runs). +# reproducible. The resulting checkpoint is a reproducible SFT baseline and a +# candidate downstream-initialization artifact; it is NOT a frozen requirement, +# and no specific checkpoint (only the base model Qwen/Qwen2.5-7B-Instruct) is +# fixed for experiments. # -# Pre-gates: contract validation (soft) + token-budget with the real model -# chat template (hard, only blocks if a row exceeds MAX_LENGTH at +# Pre-gates: contract validation (HARD: aborts on failure) + token-budget with +# the real model chat template (HARD, only blocks if a row exceeds MAX_LENGTH at # truncation=error). Every run dumps: # /hyperparams.json (this launcher's explicit knobs) # /train_config.json (verl resolved Hydra config dump) @@ -46,13 +49,21 @@ LORA_ALPHA="${LORA_ALPHA:-16}" TARGET_MODULES="${TARGET_MODULES:-all-linear}" ATTENTION_IMPL="${ATTENTION_IMPL:-sdpa}" GRAD_CKPT="${GRAD_CKPT:-true}" -SAVE_FREQ="${SAVE_FREQ:-10}" +SAVE_FREQ="${SAVE_FREQ:-50}" TEST_FREQ="${TEST_FREQ:-5}" NUM_WORKERS="${NUM_WORKERS:-0}" -MAX_CKPT_KEEP="${MAX_CKPT_KEEP:-null}" +# LoRA/FSDP checkpoints include the FULL base weights (~15GB per step for 7B), +# so keep a bounded set and a low-frequency save default to avoid filling disk. +MAX_CKPT_KEEP="${MAX_CKPT_KEEP:-4}" GPUS="${NUM_GPUS:-1}" -GRAD_ACCUM=$(( TRAIN_BATCH_SIZE / (MICRO_BATCH_SIZE_PER_GPU * GPUS) )) +_DP_EFF_BATCH=$(( MICRO_BATCH_SIZE_PER_GPU * GPUS )) +if (( TRAIN_BATCH_SIZE % _DP_EFF_BATCH != 0 )); then + echo "error: TRAIN_BATCH_SIZE ($TRAIN_BATCH_SIZE) must be divisible by" \ + "MICRO_BATCH_SIZE_PER_GPU x GPUS ($_DP_EFF_BATCH); refusing to silently truncate" >&2 + exit 2 +fi +GRAD_ACCUM=$(( TRAIN_BATCH_SIZE / _DP_EFF_BATCH )) export TOKENIZERS_PARALLELISM=false export HYDRA_FULL_ERROR=1 @@ -74,17 +85,17 @@ for split in train val test; do fi done -# ---- soft gate: contract validation report ---- -if "$PYTHON_BIN" -m script.verl.sft.validate \ +# ---- hard gate: contract validation (never train on contract-invalid data) ---- +if ! "$PYTHON_BIN" -m script.verl.sft.validate \ --dataset-dir "$DATA_DIR" \ --registry "cfg/task/registry/$DATASET.registry.json" \ --corpus "cfg/task/corpus/$DATASET.corpus.json" \ --metadata-fields field_name field_description \ --report "$OUTPUT_DIR/validate.report.json" > "$OUTPUT_DIR/validate.out" 2>&1; then - echo "[baseline] validate: valid" -else - echo "[baseline] validate: FAILED (see $OUTPUT_DIR/validate.out)" >&2 + echo "error: contract validation failed (see $OUTPUT_DIR/validate.out)" >&2 + exit 2 fi +echo "[baseline] validate: valid" # ---- hard gate: token budget ---- if ! "$PYTHON_BIN" -m script.verl.sft.check_token_budget \