# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Report a counterfactual
What the runner measured in your CI: whether an element of the constraints block changes what a tool returns, against the tool’s own noise. Positions and overlaps, never items.
# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Authorizations
nia_sk_...
Body
What a runner measured in the company's CI for one tool and one element (the tool counterfactual spec).
1 - 5000 elementsShow child attributes
Show child attributes
constraints, hard, size, exclude ^[A-Za-z][A-Za-z0-9_.:-]{0,63}$The positions compared when a recorded list has no visible_k.
1 <= x <= 100A name for the run, such as the commit or the build it ran on.
1 - 256Response
The report, computed over the completed cases.
Whether the element changes what the tool returns, beyond the tool's own noise. It never says whether
the result got better, nor how the model reacts: limits says so, and what else holds the answer
back.
Completed cases whose overlap is below their own noise.
noise_floor - overlap: how much of the first positions the element moves beyond noise.
constraints, hard, size, exclude Where the items the person engaged with went without the element, over the completed cases.
Show child attributes
Show child attributes
not_quality, model_reaction_not_measured, trivial_for_hard, few_cases, noisy_tool, cases_skipped, dry_run Mean overlap@k of the base with itself.
Mean overlap@k of the base and the variant.
The two-sided sign test of below_noise against above_noise: how likely a split this uneven is when the element changes nothing.
1 - 512The cases not completed.
Show child attributes
Show child attributes

