import weflayr
from weflayr.openapi.models import SetBenchmarkInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.set_benchmark(body=SetBenchmarkInput.from_dict({
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": ["summarize_profile"],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
],
"evaluators_impacting_benchmark": [1],
"evaluators_for_reference": [14],
"config": {
"period": {"date_from": "2026-09-24", "date_to": "2026-10-23"},
"examples": 300,
"validation_examples": 50,
},
"request_id": "summarize-profile-1",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.setBenchmark({
client,
throwOnError: true,
body: {
userEmail: 'alex@acme.com',
name: 'Summarize profile',
callNames: ['summarize_profile'],
deployments: [
{
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
],
evaluatorsImpactingBenchmark: [1],
evaluatorsForReference: [14],
config: {
period: { dateFrom: '2026-09-24', dateTo: '2026-10-23' },
examples: 300,
validationExamples: 50,
},
requestId: 'summarize-profile-1',
},
});
curl -X POST https://app.weflayr.com/api/benchmarks/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": [
"summarize_profile"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50
},
"request_id": "summarize-profile-1"
}'
{
"benchmark_uuid": "5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"url": "https://app.weflayr.com/dashboard/optimization/benchmark/?new_benchmark=1&draft=5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"name": "Summarize profile",
"status": "draft",
"message": "A saved setup that has not been started: nothing runs and no credit is spent.",
"call_names": [
"summarize_profile"
],
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"requested_at": null,
"started_at": null,
"completed_at": null,
"updated_at": "2026-10-23T09:00:00+00:00",
"estimated_cost": 0.0,
"actual_cost": null,
"credit_reserved": 0.0,
"setup": {
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"forced_deployments": [],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50,
"replays": 1,
"sampling": "smart",
"baseline_deployment": null,
"production_model_replay_deployment": null,
"model_prices_prompted_by_user": [],
"mask_pii": false,
"auto_disallow_too_costly_reasoning_efforts": true,
"ignore_max_tokens": true
}
},
"baseline": null,
"recommendations": [],
"overpriced_reasoning_efforts": [],
"recorded_for_email": "alex@acme.com",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No member of your organization has the email sam@acme.com. Ask the person for the email they use in Weflayr, and send it again.", "code": "unknown_user"}
{"error": "No benchmark of this project has the id 5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01. List the project's benchmarks with get_benchmarks."}
{"error": "Invalid request payload.", "detail": [{"field": "config.period", "error": "Field required"}]}
Benchmarks
Set benchmark
Save a benchmark setup and sample its calls.
POST
/
api
/
benchmarks
/
import weflayr
from weflayr.openapi.models import SetBenchmarkInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.set_benchmark(body=SetBenchmarkInput.from_dict({
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": ["summarize_profile"],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
],
"evaluators_impacting_benchmark": [1],
"evaluators_for_reference": [14],
"config": {
"period": {"date_from": "2026-09-24", "date_to": "2026-10-23"},
"examples": 300,
"validation_examples": 50,
},
"request_id": "summarize-profile-1",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.setBenchmark({
client,
throwOnError: true,
body: {
userEmail: 'alex@acme.com',
name: 'Summarize profile',
callNames: ['summarize_profile'],
deployments: [
{
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
],
evaluatorsImpactingBenchmark: [1],
evaluatorsForReference: [14],
config: {
period: { dateFrom: '2026-09-24', dateTo: '2026-10-23' },
examples: 300,
validationExamples: 50,
},
requestId: 'summarize-profile-1',
},
});
curl -X POST https://app.weflayr.com/api/benchmarks/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": [
"summarize_profile"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50
},
"request_id": "summarize-profile-1"
}'
{
"benchmark_uuid": "5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"url": "https://app.weflayr.com/dashboard/optimization/benchmark/?new_benchmark=1&draft=5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"name": "Summarize profile",
"status": "draft",
"message": "A saved setup that has not been started: nothing runs and no credit is spent.",
"call_names": [
"summarize_profile"
],
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"requested_at": null,
"started_at": null,
"completed_at": null,
"updated_at": "2026-10-23T09:00:00+00:00",
"estimated_cost": 0.0,
"actual_cost": null,
"credit_reserved": 0.0,
"setup": {
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"forced_deployments": [],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50,
"replays": 1,
"sampling": "smart",
"baseline_deployment": null,
"production_model_replay_deployment": null,
"model_prices_prompted_by_user": [],
"mask_pii": false,
"auto_disallow_too_costly_reasoning_efforts": true,
"ignore_max_tokens": true
}
},
"baseline": null,
"recommendations": [],
"overpriced_reasoning_efforts": [],
"recorded_for_email": "alex@acme.com",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No member of your organization has the email sam@acme.com. Ask the person for the email they use in Weflayr, and send it again.", "code": "unknown_user"}
{"error": "No benchmark of this project has the id 5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01. List the project's benchmarks with get_benchmarks."}
{"error": "Invalid request payload.", "detail": [{"field": "config.period", "error": "Field required"}]}
Save a benchmark setup (Create a benchmark explains each setting). Saving samples the calls and spends no credit: the benchmark runs once it is started with start_or_resume_benchmark.
The response also carries
Parameters
Passed in the request body.string
The email of the member of your organization this is for, recorded as the benchmark’s author, as who started it and as who gets the email when it is done. Defaults to the email of the person who created the API key, or for a key without one of the project’s creator. The response’s recorded_for_email gives the email recorded: tell the person, and send this to record another.
string
required
The benchmark’s name.
string[]
required
The call names the benchmark replays, as get_calls lists them.
object[]
The exploration deployments, as get_benchmark_candidates lists them. Each runs on a few examples first, and only those likely to meet an optimization objective run the full benchmark (steps of a benchmark).
object[]
The deployments that skip the exploration and go straight to the full benchmark, as get_benchmark_candidates lists them. A deployment is in one list only.
integer[]
The evaluators whose verdicts the recommendation counts, as get_evaluators lists them. This list and evaluators_for_reference together must include every always-used evaluator, and starting needs at least one evaluator.
integer[]
The evaluators that score and show the answers without counting in the recommendation. An evaluator is in one list only.
object
required
How the benchmark samples, replays and compares. Only period has no default.
Show config
Show config
string[]
What the benchmark recommends models for. Defaults to the first two. One of
same_quality_lowest_cost, same_cost_best_quality or best_quality.object
required
{"date_from": "YYYY-MM-DD", "date_to": "YYYY-MM-DD"}: the days the calls are sampled from. date_to defaults to today.integer
default:"300"
How many calls are replayed.
integer
default:"50"
How many calls are kept aside as the validation set, to check the recommended mix on calls not used to choose it.
integer
default:"1"
How many times each call is replayed on each model. One of
1, 3 or 5.string
default:"smart"
How the calls are split between customers. One of
smart, equal or random.object
{"model", "provider", "region"}: the deployment every candidate is compared with. Omit it to compare with the answers production logged.object
For a production baseline served by one catalog model only: the deployment that replays the calls at the other reasoning efforts, so the recommended mix can keep that model (see
production.default_replay_deployment of Get benchmark candidates).object[]
{"model", "input_per_1m", "output_per_1m"} for each production model nothing prices (see production.models_to_price of the candidates).boolean
default:"false"
Mask personal data in the replayed calls.
boolean
default:"true"
Stop testing a model at a reasoning effort that costs on average more than 6 times its medium effort (its default effort when it has no medium) on the same calls while writing at least twice the baseline’s output tokens, and leave its answers out of the results. Checked once the first 3 examples are done, then after every example. Also stop testing a model at the effort it is first tested at, and drop the model from the benchmark, when it writes at least 250,000 output tokens to a call whose logged answer wrote fewer than 10,000.
boolean
default:"true"
Replay the calls without the
max_tokens they set. A model’s reasoning tokens count toward max_tokens, so a model at a high reasoning effort can reach it before writing any answer. An answer with no text is judged worse than the baseline’s. Ignored when none of the calls of the period sets max_tokens, and then returned as null.string
The saved setup this one replaces, keeping its recorded person. Omit it to save a new setup.
string
An id you choose for this request. Sending it again after a timeout returns the setup it saved instead of saving another.
blockers (what prevents the usual next step), in plain words. It is empty when the usual workflow applies.
import weflayr
from weflayr.openapi.models import SetBenchmarkInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.set_benchmark(body=SetBenchmarkInput.from_dict({
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": ["summarize_profile"],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
],
"evaluators_impacting_benchmark": [1],
"evaluators_for_reference": [14],
"config": {
"period": {"date_from": "2026-09-24", "date_to": "2026-10-23"},
"examples": 300,
"validation_examples": 50,
},
"request_id": "summarize-profile-1",
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.setBenchmark({
client,
throwOnError: true,
body: {
userEmail: 'alex@acme.com',
name: 'Summarize profile',
callNames: ['summarize_profile'],
deployments: [
{
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
],
evaluatorsImpactingBenchmark: [1],
evaluatorsForReference: [14],
config: {
period: { dateFrom: '2026-09-24', dateTo: '2026-10-23' },
examples: 300,
validationExamples: 50,
},
requestId: 'summarize-profile-1',
},
});
curl -X POST https://app.weflayr.com/api/benchmarks/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"user_email": "alex@acme.com",
"name": "Summarize profile",
"call_names": [
"summarize_profile"
],
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50
},
"request_id": "summarize-profile-1"
}'
{
"benchmark_uuid": "5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"url": "https://app.weflayr.com/dashboard/optimization/benchmark/?new_benchmark=1&draft=5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01",
"name": "Summarize profile",
"status": "draft",
"message": "A saved setup that has not been started: nothing runs and no credit is spent.",
"call_names": [
"summarize_profile"
],
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"requested_at": null,
"started_at": null,
"completed_at": null,
"updated_at": "2026-10-23T09:00:00+00:00",
"estimated_cost": 0.0,
"actual_cost": null,
"credit_reserved": 0.0,
"setup": {
"deployments": [
{
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
],
"forced_deployments": [],
"evaluators_impacting_benchmark": [
1
],
"evaluators_for_reference": [
14
],
"config": {
"optimization_objectives": [
"same_quality_lowest_cost",
"same_cost_best_quality"
],
"period": {
"date_from": "2026-09-24",
"date_to": "2026-10-23"
},
"examples": 300,
"validation_examples": 50,
"replays": 1,
"sampling": "smart",
"baseline_deployment": null,
"production_model_replay_deployment": null,
"model_prices_prompted_by_user": [],
"mask_pii": false,
"auto_disallow_too_costly_reasoning_efforts": true,
"ignore_max_tokens": true
}
},
"baseline": null,
"recommendations": [],
"overpriced_reasoning_efforts": [],
"recorded_for_email": "alex@acme.com",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No member of your organization has the email sam@acme.com. Ask the person for the email they use in Weflayr, and send it again.", "code": "unknown_user"}
{"error": "No benchmark of this project has the id 5c1d6a52-3f5f-4c53-9a0e-6f2b9c7d8e01. List the project's benchmarks with get_benchmarks."}
{"error": "Invalid request payload.", "detail": [{"field": "config.period", "error": "Field required"}]}