Agent runs on sandboxed tasks

Put the built-in agent, or your own, to work on tasks in a sandbox, including SWE-bench.

Source: examples/agents. Read the guide for the concepts behind it.

benchmarks.yaml

# Benchmark datasets as agent runs. Each spec below is a separate file in
# practice; they are shown together here. Grading is always the benchmark's
# own code or rule.

# Terminal-Bench 2 / any Harbor dataset: a directory of task directories
# under datasets_dir. The Dockerfile is translated (FROM, RUN, COPY, ...).
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: terminal-bench}
spec:
  target:
    agent:
      cli: {command: [sh, -c, "my-terminal-agent"], timeout: 15m}
  dataset: {uri: "harbor://terminal-bench-2?tasks=hello-world,fix-git"}
  evaluators: [{ref: task-success}, {ref: code-quality}]   # code-quality uses the judge
---
# tau2-bench: needs the taubench adapter and tau2's data (TAU2_DATA_DIR).
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: tau-airline}
spec:
  target: {connector: anthropic, model: claude-opus-5-5}
  # The simulated user speaks through the judge (the server's default, or
  # --judge-* when embedded).
  dataset: {uri: "taubench://airline"}
  harness:
    external:
      python: evalsi_taubench:TauBenchHarness
      config: {maxSteps: 100}
  evaluators: [{ref: task-success}]
  trials: 4                                               # reports pass^4
---
# BFCL: needs the bfcl adapter.
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata: {name: bfcl-simple}
spec:
  target: {connector: openai-compatible, base_url: "http://localhost:8000/v1", model: my-model}
  dataset: {uri: "bfcl://simple_python"}
  harness: {external: {python: evalsi_bfcl:BFCLHarness}}
  evaluators: [{ref: task-success}]

fix-calc.jsonl

{"id": "add", "input": "calc.py has a bug: add(2, 3) returns -1. Fix it."}
{"id": "mul", "input": "calc.py has a bug: mul(2, 3) returns 5. Fix it."}

fix-calc.yaml

# An agent run with the built-in harness: a model works in a sandboxed git
# checkout with bash/read_file/write_file, and a hidden checker grades the end
# state. The same file runs embedded or on a server:
#   evalsi run -f examples/agents/fix-calc.yaml \
#     --judge-provider anthropic --judge-model claude-opus-5-5      # needs evalsid on PATH
#   evalsi run -f examples/agents/fix-calc.yaml --server http://localhost:8080
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata:
  name: fix-calc
  labels: {evals.si/project: quickstart}   # on Kubernetes, default: the namespace
spec:
  target:
    connector: anthropic
    model: claude-opus-5-5
  dataset:
    path: fix-calc.jsonl
  harness:
    builtin:
      max_steps: 20
      budget: {usd: 0.50, wall_clock: 5m}
      pricing: {input_per_mtok: 5, output_per_mtok: 25}
  environment:
    files:
      calc.py: |
        def add(a, b):
            return a - b


        def mul(a, b):
            return a + b
    setup:
      - git init -q && git add -A && git -c user.email=e@x -c user.name=e commit -qm base
    sandbox: {min_isolation: namespaced, network: deny}
    checker:
      # Written after the agent finishes, so the agent never sees it.
      files:
        test_hidden.py: |
          from calc import add, mul
          assert add(2, 3) == 5 and mul(2, 3) == 6
      command: [python3, test_hidden.py]
  evaluators:
    - ref: task-success          # the checker's verdict; with trials, pass@k and pass^k
    - ref: code-quality          # a judge reviews the agent's diff
    - ref: policy-violations     # sandbox denials, refused egress, escalation requests
    - ref: agent-efficiency
  # code-quality needs a judge: --judge-* options (or EVALSI_JUDGE_*) when
  # embedded; on a server its default judge, or one named with `judge:`.
  trials: 3
  gates:
    - {metric: task-success.pass^3, min: 0.5}

swebench-verified.yaml

# The Phase 3 exit criterion: SWE-bench Verified instances with a
# bring-your-own CLI agent, in the strongest sandbox rung on the host
# (Firecracker with KVM, otherwise bubblewrap); grading is the swebench
# package's own, so it is identical on every rung.
#
# The worker needs the SWE-bench adapter (python/adapters/swebench). Put
# Verified's test.parquet under the server's datasets_dir, for example
# datasets_dir/swebench-verified/test.parquet, then:
#   evalsi run -f examples/agents/swebench-verified.yaml --server http://localhost:8080
apiVersion: evals.si/v1alpha1
kind: EvalRun
metadata:
  name: swebench-verified-subset
spec:
  target:
    agent:
      cli:
        # Your agent, run inside the instance's image in /testbed. It gets the
        # issue on stdin; EVALSI_WORKDIR names the working directory.
        command: [sh, -c, "my-coding-agent --non-interactive"]
        install: ["pip install my-coding-agent"]
        env_from: {MY_AGENT_API_KEY: MY_AGENT_API_KEY}
        allow_hosts: [api.example.com]
        timeout: 30m
  environment:
    # Merged under each instance's environment: `install` runs during setup
    # (once, then snapshotted), which needs the network; the agent itself only
    # reaches allow_hosts.
    setup_network: allow
  dataset:
    uri: "swebench://swebench-verified?split=test&instances=django__django-11099,astropy__astropy-12907,sympy__sympy-20590"
  evaluators:
    - ref: task-success          # resolved per the official FAIL_TO_PASS / PASS_TO_PASS rule
    - ref: policy-violations
  trials: 1
  gates:
    - {metric: task-success, min: 0.3}