Server configuration

A baseline evalsid configuration.

Source: examples/server. Read the guide for the concepts behind it.

evalsi.yaml

# evalsid configuration. Start with:
#   evalsi serve --config examples/server/evalsi.yaml
listen: 127.0.0.1:8080
# Loopback without an auth section serves unauthenticated. Any other address
# needs authentication: see examples/auth/evalsi.yaml and
# docs/guides/identity.md (or `auth: {mode: none}` to opt out explicitly).
data_dir: .evalsi            # SQLite database for runs, traces and policies

# Run specs may name dataset files relative to this directory (they cannot
# escape it). Leave unset to accept only inline records and hf:// datasets.
# datasets_dir: ./datasets
# datasets_dir: s3://my-bucket/evalsi/datasets   # with storage.s3

# External storage, for several replicas (and Kubernetes). Without it,
# everything is in SQLite under data_dir.
# storage:
#   postgres: {dsn_env: EVALSI_POSTGRES_DSN}        # metadata, runs, results
#   clickhouse:                                    # traces and online scores
#     url: http://clickhouse:8123
#     database: evalsi
#     user: evalsi
#     password_env: EVALSI_CLICKHOUSE_PASSWORD
#   s3:                                            # an s3:// datasets_dir
#     endpoint: s3.us-east-1.amazonaws.com         # or minio:9000 with insecure and path_style
#     region: us-east-1                            # credentials: the AWS chain (IRSA, ...)

# Several replicas and worker processes (Kubernetes): work goes through NATS
# JetStream queues per pool, served by `evalsid worker --pools ...`. API
# replicas share storage.postgres; runs survive a replica stopping (another
# adopts them), and one replica at a time is elected the policy engine.
# cluster:
#   url: nats://nats:4222
#   # embedded: {listen: 127.0.0.1:4222}    # or run NATS inside this evalsid
# worker:
#   pools: [cpu, judge]                       # this process also serves these pools

worker:
  # Runs the evalsi CLI; evalsid appends "worker --listen unix://...".
  # Use the interpreter of the environment where evalsi[server] is installed.
  command: ["python", "-m", "evalsi"]
  start_timeout: 60s

# Judges are referenced by name in requests. API keys are read by the worker
# from the named environment variables; they are never stored here.
judges:
  claude:
    provider: anthropic
    model: claude-opus-5-5
    effort: low
    requests_per_minute: 300
    # projects: [support, research]    # only these projects may use it (default: all)
  local:
    provider: openai-compatible
    model: my-judge
    base_url: http://localhost:8000/v1
default_judge: claude

# Worker variables that run specs, agents and evaluator params may name
# (api_key_env, headers_env, env_from, params ending in _env), per project
# and optionally per destination host. Checked whenever auth is enabled or a
# grant is listed. See docs/guides/identity.md.
# Reloaded without a restart (every 15s and on SIGHUP), with judges' projects.
# credentials:
#   grants:
#     - {env: OPENAI_API_KEY, projects: ["*"], hosts: [api.openai.com]}   # HTTPS only
#     - {env: LAB_KEY, projects: [research], hosts: [vllm.lab], allow_http: true}
#     - {env: SUPPORT_AGENT_TOKEN, projects: [support]}

evaluate:
  batch_size: 32      # records per worker call
  parallelism: 8      # concurrent worker calls per request
  max_records: 10000  # larger jobs use EvaluateStream

runs:
  max_concurrent: 4   # runs executing at once; others wait as PENDING
  # Finished runs older than this are deleted hourly, as DeleteRun does (records,
  # outputs, results). A run labelled evalsi.keep=true is never deleted. Unset: keep forever.
  # retention: 2160h
  # project_retention: {research: 8760h, audit: "0"}   # "0": that project's runs are kept
  # Finished runs older than this are archived hourly (ArchiveRun): records and results go,
  # the summary stays until retention deletes it. Must be shorter than retention.
  # archive_after: 168h
  # project_archive_after: {audit: "0"}                # "0": never archived

# OTLP is always served on the main port (gRPC TraceService/Export and HTTP
# POST /v1/traces). These also open the standard ports for SDK defaults.
otlp:
  grpc_listen: 127.0.0.1:4317
  http_listen: 127.0.0.1:4318
  grace: 2s           # wait for late spans after a trace's root span ends
  max_traces: 10000

traces:
  retention: 168h

# Online evaluation policies applied at startup (OnlineEvalPolicy JSON form).
# Policies can also be managed at runtime with `evalsi policy apply`.
policies:
  - name: all-agents-latency
    stages:
      - evaluators: [{ref: builtin/latency}]

# Isolation for code-executing evaluators (builtin/unit-tests). Rungs are tried
# strongest first; nothing runs unconfined. `evalsid sandbox probe` shows which
# rungs work on this host.
sandbox:
  ladder: [bwrap, landlock]
  min_isolation: confined
  # Interpreters or toolchains outside /usr that sandboxed code may use.
  read_only_paths: []

# Where finished runs and online scores are exported. Exports run in the
# background with retries and never fail a run.
sinks: []
#  - mlflow:
#      tracking_uri: http://mlflow:5000
#      experiment: support-agent        # default: the run's project, else "evalsi"
#      token_env: MLFLOW_TRACKING_TOKEN # optional bearer token
#      trace_feedback: true             # online scores become assessments on MLflow traces
#  - otel:
#      endpoint: http://otel-collector:4318   # gen_ai.evaluation.result events on /v1/logs
#      headers_env: {x-api-key: OTEL_API_KEY}
#  - langfuse:
#      host: https://cloud.langfuse.com       # runs become traces with a score per metric
#      public_key_env: LANGFUSE_PUBLIC_KEY
#      secret_key_env: LANGFUSE_SECRET_KEY
#      trace_feedback: true                   # online scores onto the same trace IDs in Langfuse
#  - phoenix:
#      endpoint: http://phoenix:6006          # online scores as trace annotations
#      api_key_env: PHOENIX_API_KEY

# Trace sources (docs/guides/trace-sources.md): stores whose traces evalsid
# reads, scores and writes back to. Their endpoints must be https to a public
# name unless listed here.
sources: {}
#  dir: /etc/evalsi/sources          # where Secrets are mounted, for credentials.file
#  allow_hosts: [mlflow.studio.svc.cluster.local, "*.corp.example.com"]
#  max_records_per_second: 200        # for a source that sets no cap

# The Reward Service (rewards for RL trainers; docs/guides/fine-tuning.md).
rewards:
  max_rollouts: 16384     # per ScoreRewards call
  max_inflight: 65536     # across calls; further calls wait (back-pressure)
  cache_size: 1048576     # component scores kept in memory

# Per-project quotas; 0 or unset means unlimited. `projects` overrides the
# default field by field.
quotas:
  default: {}
#   max_concurrent_runs: 4        # further runs wait in PENDING (per replica)
#   max_stored_runs: 10000        # CreateRun fails past it
#   judge_tokens_per_day: 5000000 # UTC day, across the project's runs
#   target_tokens_per_day: 20000000
#   sandbox_seconds_per_day: 36000 # UTC day: agent tasks (start to end) and code evaluators, summed
#   max_reward_rollouts: 8192     # Reward Service rollouts in flight (per replica)
# projects:
#   research: {judge_tokens_per_day: 50000000}