{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://evalship.com/schema/v1.json",
  "title": "evalship results v1",
  "description": "Eval results for evalship. Save as *.json in the evalship-results artifact.",
  "type": "object",
  "required": ["cases"],
  "properties": {
    "schema": { "const": "evalship/v1" },
    "suite": { "type": "string", "maxLength": 100, "description": "Default suite for cases without one." },
    "cases": {
      "type": "array",
      "items": {
        "type": "object",
        "anyOf": [ { "required": ["key"] }, { "required": ["name"] } ],
        "properties": {
          "key": { "type": "string", "description": "Stable identifier; keep it the same across runs." },
          "name": { "type": "string" },
          "suite": { "type": "string" },
          "passed": { "type": "boolean", "description": "Required unless score and threshold are both given." },
          "score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
          "threshold": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
          "duration_ms": { "type": ["number", "null"] },
          "cost_usd": { "type": ["number", "null"] },
          "tokens": { "type": ["integer", "null"] },
          "input": { "type": ["string", "null"], "description": "Truncated to 4 KB." },
          "output": { "type": ["string", "null"], "description": "Truncated to 4 KB." },
          "failure": { "type": ["string", "null"], "description": "Why it failed. Truncated to 4 KB." },
          "file": { "type": ["string", "null"] },
          "targets": { "type": "array", "items": { "type": "string" }, "description": "Code this case exercises: path or path#symbol." },
          "metadata": { "type": "object" }
        }
      }
    }
  }
}