import weflayr
from weflayr.openapi.models import GetBenchmarkCandidatesInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_benchmark_candidates(body=GetBenchmarkCandidatesInput.from_dict({
"call_names": ["summarize_profile"],
"date_from": "2026-09-24",
"date_to": "2026-10-23",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getBenchmarkCandidates({
client,
throwOnError: true,
body: {
callNames: ['summarize_profile'],
dateFrom: '2026-09-24',
dateTo: '2026-10-23',
},
});
curl -X POST https://app.weflayr.com/api/benchmark-candidates/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"call_names": [
"summarize_profile"
],
"date_from": "2026-09-24",
"date_to": "2026-10-23"
}'
{
"required_capabilities": [
"tool-use"
],
"largest_call_input_tokens": 3200,
"production": {
"models": [
{
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"calls": 1840,
"input_per_1m": 2.0,
"output_per_1m": 10.0
}
],
"models_to_price": [],
"single_catalog_model": "claude-sonnet-5-5",
"default_replay_deployment": {
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"region": "GLOBAL"
}
},
"baseline_unavailable_message": null,
"models": [
{
"name": "gpt-6-luna",
"built_by": "openai",
"max_input_tokens": 922000,
"released": "2026-09",
"capabilities": [
"tool-use",
"reasoning"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
"input_per_1m": 0.1,
"output_per_1m": 0.5,
"cache_read_per_1m": 0.01,
"missing_capabilities": [],
"recommended": true,
"blocked_from_exploration": [],
"blocked_from_always_benchmark": []
}
]
}
],
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "The gateway serves no deployment named (model=gpt-9, provider=openai, region=GLOBAL). Pick deployments the AI gateway's GET /v1/models lists.", "code": "unknown_deployment"}
{"error": "Invalid request payload.", "detail": [{"field": "date_from", "error": "Field required"}]}
Benchmarks
Get benchmark candidates
Get the models a benchmark of some calls can run on.
POST
/
api
/
benchmark-candidates
/
import weflayr
from weflayr.openapi.models import GetBenchmarkCandidatesInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_benchmark_candidates(body=GetBenchmarkCandidatesInput.from_dict({
"call_names": ["summarize_profile"],
"date_from": "2026-09-24",
"date_to": "2026-10-23",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getBenchmarkCandidates({
client,
throwOnError: true,
body: {
callNames: ['summarize_profile'],
dateFrom: '2026-09-24',
dateTo: '2026-10-23',
},
});
curl -X POST https://app.weflayr.com/api/benchmark-candidates/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"call_names": [
"summarize_profile"
],
"date_from": "2026-09-24",
"date_to": "2026-10-23"
}'
{
"required_capabilities": [
"tool-use"
],
"largest_call_input_tokens": 3200,
"production": {
"models": [
{
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"calls": 1840,
"input_per_1m": 2.0,
"output_per_1m": 10.0
}
],
"models_to_price": [],
"single_catalog_model": "claude-sonnet-5-5",
"default_replay_deployment": {
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"region": "GLOBAL"
}
},
"baseline_unavailable_message": null,
"models": [
{
"name": "gpt-6-luna",
"built_by": "openai",
"max_input_tokens": 922000,
"released": "2026-09",
"capabilities": [
"tool-use",
"reasoning"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
"input_per_1m": 0.1,
"output_per_1m": 0.5,
"cache_read_per_1m": 0.01,
"missing_capabilities": [],
"recommended": true,
"blocked_from_exploration": [],
"blocked_from_always_benchmark": []
}
]
}
],
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "The gateway serves no deployment named (model=gpt-9, provider=openai, region=GLOBAL). Pick deployments the AI gateway's GET /v1/models lists.", "code": "unknown_deployment"}
{"error": "Invalid request payload.", "detail": [{"field": "date_from", "error": "Field required"}]}
Get the models a benchmark of these calls can run on, with each deployment’s price per 1M tokens, the ones recommended for the optimization objectives, and why the others are blocked: a capability your calls used that the model lacks, an input limit smaller than your largest call, or a price the objectives cannot route to. Also returns the models production served the calls with, and their price. Then save the setup with set_benchmark, with the recommended deployments that are not blocked and evaluators from get_evaluators.
The response also carries
Parameters
Passed in the request body.string[]
required
The call names the benchmark would replay, as get_calls lists them.
string
required
Start of the period, inclusive. YYYY-MM-DD.
string
End of the period, inclusive. YYYY-MM-DD. Defaults to today.
string[]
The optimization objectives the benchmark would run, which decide what is recommended and what is blocked. Defaults to the first two. One of
same_quality_lowest_cost, same_cost_best_quality or best_quality.object
The deployment the candidates would be compared with. Omit it to compare with the answers production logged.
object[]
The price of each production model nothing prices.
blockers (what prevents the usual next step), in plain words. It is empty when the usual workflow applies.
import weflayr
from weflayr.openapi.models import GetBenchmarkCandidatesInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_benchmark_candidates(body=GetBenchmarkCandidatesInput.from_dict({
"call_names": ["summarize_profile"],
"date_from": "2026-09-24",
"date_to": "2026-10-23",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getBenchmarkCandidates({
client,
throwOnError: true,
body: {
callNames: ['summarize_profile'],
dateFrom: '2026-09-24',
dateTo: '2026-10-23',
},
});
curl -X POST https://app.weflayr.com/api/benchmark-candidates/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"call_names": [
"summarize_profile"
],
"date_from": "2026-09-24",
"date_to": "2026-10-23"
}'
{
"required_capabilities": [
"tool-use"
],
"largest_call_input_tokens": 3200,
"production": {
"models": [
{
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"calls": 1840,
"input_per_1m": 2.0,
"output_per_1m": 10.0
}
],
"models_to_price": [],
"single_catalog_model": "claude-sonnet-5-5",
"default_replay_deployment": {
"model": "claude-sonnet-5-5",
"provider": "anthropic",
"region": "GLOBAL"
}
},
"baseline_unavailable_message": null,
"models": [
{
"name": "gpt-6-luna",
"built_by": "openai",
"max_input_tokens": 922000,
"released": "2026-09",
"capabilities": [
"tool-use",
"reasoning"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
"input_per_1m": 0.1,
"output_per_1m": 0.5,
"cache_read_per_1m": 0.01,
"missing_capabilities": [],
"recommended": true,
"blocked_from_exploration": [],
"blocked_from_always_benchmark": []
}
]
}
],
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "The gateway serves no deployment named (model=gpt-9, provider=openai, region=GLOBAL). Pick deployments the AI gateway's GET /v1/models lists.", "code": "unknown_deployment"}
{"error": "Invalid request payload.", "detail": [{"field": "date_from", "error": "Field required"}]}