Eval runs

Create an eval run (async)

Creates a suite run from an existing suiteId (rerun) and/or inline tests, then detaches execution and responds 202 immediately with the runId. Validation and quota errors surface on this request; poll GET /eval-runs/{runId} for progress. The run appears live in the hosted UI Runs tab, tagged source: "api".

A bare suiteId with no inline tests reruns the suite as configured. Per-organization concurrency is capped (default 2 concurrent runs); exceeding it returns 429 with details.reason: "CONCURRENT_RUN_LIMIT".

For a suite with attached project environments, pass environmentId to choose which one the run uses; the 202 echoes the resolved environment triple, and GET /eval-runs/{runId} reports the same triple for the life of the run.

POST

projects

{projectId}

eval-runs

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  },
  "environment": null
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Option 1
Option 2

Two valid shapes: suiteId (rerun an existing suite, optionally upserting inline tests into it) or suiteName + tests + serverIds (create a new suite and run it). Inline tests alone — without a suiteId or a suiteName — are rejected with VALIDATION_ERROR.

environmentId requires suiteId: an environment is launchable only through a suite that has it attached (environmentIds, set via PATCH /eval-suites/{suiteId}), so an environment run on a not-yet-created suite could never be satisfied. environmentId and serverIds are mutually exclusive.

suiteId

string

required

Existing suite to rerun. A bare suiteId with no tests reruns the suite exactly as configured.

suiteName

string

Name for a new suite. Required (non-empty) when no suiteId is given.

suiteDescription

string

tests

object[]

Inline test cases to upsert into the suite before running.

Maximum array length: 100

Show child attributes

serverIds

string[]

Servers (by ID) the run connects to. Required when creating a new suite; optional on reruns — when omitted, the run connects the suite's saved server selection (the set its snapshot references). A rerun of a suite with no saved selection is rejected with VALIDATION_ERROR (details.reason: "NO_SAVED_SERVER_SELECTION"). Rejected outright for a suite with attached environments (details.reason: "ENVIRONMENT_SERVERS_NOT_OVERRIDABLE"): the environment supplies a closed set that a server override cannot change, so accepting one would connect a different set than the run is stamped with.

Minimum array length: 1

modelApiKeys

object

Optional per-provider model API keys (e.g. { "anthropic": "sk-ant-…" }). Falls back to your organization's configured providers when omitted.

Show child attributes

notes

string

passCriteria

object

Show child attributes

iterationOverride

integer

Override the per-case runs count for this run only.

Required range: 1 <= x <= 10

environmentId

string

Run against one of the suite's attached project environments. Requires suiteId, and must be a member of that suite's environmentIds — otherwise 400 with details.reason: "ENVIRONMENT_NOT_ATTACHED", raised before any case is authored or any server connected.

Omission is meaningful: a suite with no attached environments runs legacy; a suite with exactly ONE attached environment runs against it automatically (the response's environment says which); a suite with several returns 400 with details.reason: "ENVIRONMENT_REQUIRED", naming the candidates.

The environment supplies the closed server set (so serverIds is not required, and is rejected alongside it), and the run is pinned to the revision resolved at launch — if the environment changes in between, the run is rejected with 409 rather than executing against a different configuration.

Response

Run created; execution continues in the background.

runId

string

required

suiteId

string

required

status

string

required

Allowed value: "running"

caseUpsert

object

required

Per-case upsert outcomes for inline tests. Partial failures don't abort the run.

Show child attributes

servers

object[]

The servers the run connects to — explicit or derived from the suite's saved selection. name is present when known (always, on the derived path).

Show child attributes

environment

object | null

The project environment a run is pinned to, at the revision resolved when it launched. null for a legacy run that used the suite's saved server selection.

Show child attributes

⌘I

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  },
  "environment": null
}

API Guides

Endpoints

Create an eval run (async)

Authorizations

Path Parameters

Body

Response