import weflayr
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_evaluators(limit=20)
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getEvaluators({
client,
throwOnError: true,
query: {
limit: 20,
},
});
curl "https://app.weflayr.com/api/evaluators/?limit=20" \
-H "Authorization: Bearer $WEFLAYR_API_KEY"
{
"evaluators": [
{
"evaluator_id": 1,
"name": "Overall quality",
"kind": "judge",
"instruction": "The answer is as good as the baseline's.",
"builtin": true,
"always_used": true,
"in_library": false,
"generated": false
},
{
"evaluator_id": 14,
"name": "Polite tone",
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"builtin": false,
"always_used": false,
"in_library": true,
"generated": false
}
],
"next_cursor": null,
"has_more": false,
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "Invalid request payload.", "detail": [{"field": "limit", "error": "Input should be less than or equal to 1000"}]}
Evaluators
Get evaluators
Get the evaluators a benchmark can score with.
GET
/
api
/
evaluators
/
import weflayr
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_evaluators(limit=20)
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getEvaluators({
client,
throwOnError: true,
query: {
limit: 20,
},
});
curl "https://app.weflayr.com/api/evaluators/?limit=20" \
-H "Authorization: Bearer $WEFLAYR_API_KEY"
{
"evaluators": [
{
"evaluator_id": 1,
"name": "Overall quality",
"kind": "judge",
"instruction": "The answer is as good as the baseline's.",
"builtin": true,
"always_used": true,
"in_library": false,
"generated": false
},
{
"evaluator_id": 14,
"name": "Polite tone",
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"builtin": false,
"always_used": false,
"in_library": true,
"generated": false
}
],
"next_cursor": null,
"has_more": false,
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "Invalid request payload.", "detail": [{"field": "limit", "error": "Input should be less than or equal to 1000"}]}
Get the evaluators you can use in a benchmark, by id: the built-in ones Weflayr wrote, and the ones of your project. An always-used evaluator is in every benchmark. Write your own with set_evaluator, or have AI write them for a saved setup with generate_evaluators.
The response also carries
Parameters
string
The route is paginated: pass the previous page’s
next_cursor, as is, to get the next page. has_more says whether another page exists.integer
default:"100"
Number of evaluators per page. Max
1000.blockers (what prevents the usual next step), in plain words. It is empty when the usual workflow applies.
import weflayr
client = weflayr.client() # reads WEFLAYR_API_KEY
client.get_evaluators(limit=20)
const weflayr = require('weflayr');
const client = weflayr.client(); // reads WEFLAYR_API_KEY
await weflayr.api.getEvaluators({
client,
throwOnError: true,
query: {
limit: 20,
},
});
curl "https://app.weflayr.com/api/evaluators/?limit=20" \
-H "Authorization: Bearer $WEFLAYR_API_KEY"
{
"evaluators": [
{
"evaluator_id": 1,
"name": "Overall quality",
"kind": "judge",
"instruction": "The answer is as good as the baseline's.",
"builtin": true,
"always_used": true,
"in_library": false,
"generated": false
},
{
"evaluator_id": 14,
"name": "Polite tone",
"kind": "judge",
"instruction": "The answer is polite and never blames the customer.",
"builtin": false,
"always_used": false,
"in_library": true,
"generated": false
}
],
"next_cursor": null,
"has_more": false,
"blockers": []
}
{"error": "Invalid or missing API key. Send it as \"Authorization: Bearer <your key>\"."}
{"error": "Invalid request payload.", "detail": [{"field": "limit", "error": "Input should be less than or equal to 1000"}]}