Python
# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Outcomes and experiments
Report a counterfactual
What the runner measured in your CI: whether an element of the constraints block changes what a tool returns, against the tool’s own noise. Positions and overlaps, never items.
POST
/
v1
/
measure
/
counterfactual-runs
Python
# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Body
application/json
What a runner measured in the company's CI for one tool and one element (the tool counterfactual spec).
Response
The report, computed over the completed cases.
Whether the element changes what the tool returns, beyond the tool's own noise. It never says whether
the result got better, nor how the model reacts: limits says so, and what else holds the answer
back.

