import weflayr
from weflayr.openapi.models import TestEvaluatorInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.test_evaluator(body=TestEvaluatorInput.from_dict({
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.testEvaluator({
client,
throwOnError: true,
body: {
kind: 'judge',
instruction: 'The answer is polite and never blames the customer.',
callName: 'summarize_profile',
candidateDeployment: {
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
},
});
curl -X POST https://app.weflayr.com/api/evaluator-test/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
}'
{
"call_id": "evt_8f2a",
"candidate_output": "Thank you for reaching out. Here is a summary of the profile.",
"baseline_output": null,
"verdict": "same",
"justification": "Both answers are polite.",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No call named 'summarize_profile' with its prompt was logged in the last 30 days. List the call names with get_calls.", "code": "no_call_to_test"}
{"error": "Your organization has no credit left. Top up on the Credits page, then call this again.", "code": "insufficient_credits"}
{"error": "Invalid request payload.", "detail": [{"field": "kind", "error": "Field required"}]}
Evaluators
Test evaluator
Try an evaluator on a logged call.
POST
/
api
/
evaluator-test
/
import weflayr
from weflayr.openapi.models import TestEvaluatorInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.test_evaluator(body=TestEvaluatorInput.from_dict({
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.testEvaluator({
client,
throwOnError: true,
body: {
kind: 'judge',
instruction: 'The answer is polite and never blames the customer.',
callName: 'summarize_profile',
candidateDeployment: {
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
},
});
curl -X POST https://app.weflayr.com/api/evaluator-test/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
}'
{
"call_id": "evt_8f2a",
"candidate_output": "Thank you for reaching out. Here is a summary of the profile.",
"baseline_output": null,
"verdict": "same",
"justification": "Both answers are polite.",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No call named 'summarize_profile' with its prompt was logged in the last 30 days. List the call names with get_calls.", "code": "no_call_to_test"}
{"error": "Your organization has no credit left. Top up on the Credits page, then call this again.", "code": "insufficient_credits"}
{"error": "Invalid request payload.", "detail": [{"field": "kind", "error": "Field required"}]}
Try an unsaved evaluator on one logged call of a call name: the candidate deployment answers the call again, and the evaluator scores its answer against the baseline’s, which is the answer production logged unless you name a baseline deployment. If the verdict is what you expect, save the evaluator with set_evaluator and use it in a benchmark.
The response also carries
Spends a little credit.
Parameters
Passed in the request body.string
required
“judge” or “deterministic”.
string
required
The judge’s instruction in words, or the check’s Python code.
string
required
A call name, as get_calls lists them: the test runs on one logged call of that name, drawn at random from the last 30 days.
object
required
The deployment that answers the call again, to be scored.
object
The deployment whose answer the candidate is compared with. Omit it to compare with the answer production logged.
blockers (what prevents the usual next step), in plain words. It is empty when the usual workflow applies.
import weflayr
from weflayr.openapi.models import TestEvaluatorInput
client = weflayr.client() # reads WEFLAYR_API_KEY
client.test_evaluator(body=TestEvaluatorInput.from_dict({
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL",
},
}))
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.testEvaluator({
client,
throwOnError: true,
body: {
kind: 'judge',
instruction: 'The answer is polite and never blames the customer.',
callName: 'summarize_profile',
candidateDeployment: {
model: 'gpt-6-luna',
provider: 'openai',
region: 'GLOBAL',
},
},
});
curl -X POST https://app.weflayr.com/api/evaluator-test/ \
-H "Authorization: Bearer $WEFLAYR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"call_name": "summarize_profile",
"candidate_deployment": {
"model": "gpt-6-luna",
"provider": "openai",
"region": "GLOBAL"
}
}'
{
"call_id": "evt_8f2a",
"candidate_output": "Thank you for reaching out. Here is a summary of the profile.",
"baseline_output": null,
"verdict": "same",
"justification": "Both answers are polite.",
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "No call named 'summarize_profile' with its prompt was logged in the last 30 days. List the call names with get_calls.", "code": "no_call_to_test"}
{"error": "Your organization has no credit left. Top up on the Credits page, then call this again.", "code": "insufficient_credits"}
{"error": "Invalid request payload.", "detail": [{"field": "kind", "error": "Field required"}]}