Fine-tuning and RL rewards

Evaluators as RL rewards, and checkpoint checks for forgetting and safety regressions.

Source: examples/finetuning. Read the guide for the concepts behind it.

code-reward.yaml

# A reward for GRPO on code: the completion must contain a fenced Python
# block (a gate: no block, reward 0), then the fraction of test cases its
# program passes in the sandbox. Rollouts carry their test cases in a
# `tests` column ([{input, output}, ...]) or MBPP-style `test_list`.
#
#   evalsi rewards score --spec code-reward.yaml --data rollouts.jsonl
#   python -c "from evalsi.rewards.trl import reward_funcs; print(reward_funcs('code-reward.yaml'))"
apiVersion: evals.si/v1alpha1
kind: RewardSpec
metadata: {name: code-grpo}
spec:
  components:
    - ref: format-check
      name: format
      weight: 0.1
      gate: true
      params: {pattern: "(?s).*```python\\n.*```.*"}
    - ref: code-exec-tests
      name: tests
      weight: 0.9
      params: {timeout_s: 5}
      sandbox: {minIsolation: namespaced}
  cache: {enabled: true}

forgetting.yaml

# Catastrophic forgetting: general capability against the base model.
# Run it on every checkpoint; `evalsi checkpoints curve` compares each one
# with the base model, record by record, and fails on a significant drop.
#
#   evalsi checkpoints eval Qwen/Qwen3-8B --step base -f forgetting.yaml --training-run my-run
#   evalsi checkpoints watch out/ -f forgetting.yaml --training-run my-run \
#       --serve lora --base-url http://localhost:8000/v1 --regression math-equiv:0.02
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: forgetting}
spec:
  target:
    connector: openai-compatible      # replaced by the served checkpoint
    model: base
    base_url: http://localhost:8000/v1
    max_tokens: 2048
    temperature: 0
  dataset:
    uri: hf://openai/gsm8k?config=main&split=test
    mapping: {input: question, reference: answer}
    limit: 500
  evaluators:
    - ref: math-equiv

math-reward.yaml

# A reward for math RL (GSM8K, MATH): <think>/<answer> format as a small
# shaped bonus, the final answer's correctness, and DAPO's soft overlong
# penalty (skipped for rollouts without completion_tokens or completion_ids,
# which TRL passes). The reference answer comes from the `answer` (or `ground_truth`,
# `solution`) column; GSM8K's "#### 72" is understood.
apiVersion: evals.si/v1alpha1
kind: RewardSpec
metadata: {name: math-grpo}
spec:
  components:
    - {ref: format-check, name: format, weight: 0.1}
    - {ref: math-equiv, name: correct, weight: 1.0}
    - ref: overlong-penalty
      name: overlong
      weight: 1.0
      params: {max_length: 4096, buffer: 512}

rollouts.jsonl

{"id": "r1", "prompt": "Read n and print 2n.", "completion": "```python\nprint(int(input()) * 2)\n```", "tests": [{"input": "2", "output": "4"}, {"input": "7", "output": "14"}]}
{"id": "r2", "prompt": "Read n and print 2n.", "completion": "```python\nprint(int(input()) + 2)\n```", "tests": [{"input": "2", "output": "4"}, {"input": "7", "output": "14"}]}
{"id": "r3", "prompt": "Read n and print 2n.", "completion": "print(int(input()) * 2)", "tests": [{"input": "2", "output": "4"}, {"input": "7", "output": "14"}]}

safety-regression.yaml

# Safety regression: refusal on harmful prompts should not fall, and
# over-refusal on benign ones should not rise. Each prompt carries
# metadata.kind (harmful or benign); compare checkpoints with the base
# model through `evalsi checkpoints curve`.
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: safety-regression}
spec:
  target:
    connector: openai-compatible
    model: base
    base_url: http://localhost:8000/v1
    max_tokens: 1024
    temperature: 0
  dataset:
    path: safety-prompts.jsonl        # {"input": ..., "metadata": {"kind": "harmful"}}
  evaluators:
    - ref: refusal
    - ref: harmlessness               # needs a judge (EVALSI_JUDGE_* or a server judge)

sampling.yaml

# Sampling behaviour and mode collapse: several samples per prompt at
# temperature 1 give pass@k and pass^k, and the diversity of the outputs.
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: sampling}
spec:
  target:
    connector: openai-compatible
    model: base
    base_url: http://localhost:8000/v1
    max_tokens: 2048
    temperature: 1.0
  dataset:
    uri: hf://openai/gsm8k?config=main&split=test
    mapping: {input: question, reference: answer}
    limit: 200
  evaluators:
    - ref: math-equiv                 # with trials: math-equiv.pass@8 and pass^8
    - ref: diversity
  trials: 8