# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Reportar um contrafactual
O que o executor mediu no seu CI: se um elemento do bloco de restrições muda o que uma ferramenta devolve, contra o ruído da própria ferramenta. Posições e sobreposições, nunca itens.
# niadra counterfactual --tools app.tools:TOOLS --tool search_products --element hard --scenario sc_01J9...
# Runs each recorded call live, with and without the element, and reports positions and overlaps only// The Python SDK's `niadra counterfactual` command runs the cases; the report is read here
const report = await niadra.api.counterfactualRunRead(runId);
console.log(report.effect, report.limits);curl --request POST \
--url https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"call_id": "<string>",
"turn_id": "<string>",
"base_count": 1,
"dry_run": false,
"engaged": [
{
"base": 5000,
"variant": 5000
}
],
"k": 50,
"noise": 0.5,
"overlap": 0.5,
"variant_count": 1
}
],
"tool": "<string>",
"k": 10,
"label": "<string>"
}
'const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
call_id: '<string>',
turn_id: '<string>',
base_count: 1,
dry_run: false,
engaged: [{base: 5000, variant: 5000}],
k: 50,
noise: 0.5,
overlap: 0.5,
variant_count: 1
}
],
tool: '<string>',
k: 10,
label: '<string>'
})
};
fetch('https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'call_id' => '<string>',
'turn_id' => '<string>',
'base_count' => 1,
'dry_run' => false,
'engaged' => [
[
'base' => 5000,
'variant' => 5000
]
],
'k' => 50,
'noise' => 0.5,
'overlap' => 0.5,
'variant_count' => 1
]
],
'tool' => '<string>',
'k' => 10,
'label' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{space}.{region}.api.niadra.com/v1/measure/counterfactual-runs")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"call_id\": \"<string>\",\n \"turn_id\": \"<string>\",\n \"base_count\": 1,\n \"dry_run\": false,\n \"engaged\": [\n {\n \"base\": 5000,\n \"variant\": 5000\n }\n ],\n \"k\": 50,\n \"noise\": 0.5,\n \"overlap\": 0.5,\n \"variant_count\": 1\n }\n ],\n \"tool\": \"<string>\",\n \"k\": 10,\n \"label\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"above_noise": 123,
"below_noise": 123,
"cases": 123,
"completed": 123,
"created_at": "2023-11-07T05:31:56Z",
"effect": 123,
"element": "constraints",
"engaged": {
"gained": 123,
"items": 123,
"kept": 123,
"lost": 123,
"shown": 123,
"mean_shift": 123
},
"limits": [
"not_quality"
],
"noise_floor": 123,
"overlap": 123,
"p_value": 123,
"run_id": "<string>",
"ties": 123,
"tool": "<string>",
"label": "<string>",
"skipped": {}
}{
"code": "<string>",
"status": 123,
"title": "<string>",
"detail": "<string>",
"request_id": "<string>",
"type": "about:blank"
}Autorizações
nia_sk_...
Corpo
O que um executor mediu no CI da empresa para uma ferramenta e um elemento (a especificação do contrafactual de ferramenta).
1 - 5000 elementsShow child attributes
Show child attributes
constraints, hard, size, exclude ^[A-Za-z][A-Za-z0-9_.:-]{0,63}$As posições comparadas quando uma lista gravada não tem visible_k.
1 <= x <= 100Um nome para a execução, como o commit ou a build em que rodou.
1 - 256Resposta
O relatório, calculado sobre os casos concluídos.
Se o elemento muda o que a ferramenta devolve, além do ruído da própria ferramenta. Nunca diz se o resultado melhorou, nem como o modelo reage: limits diz isso, e o que mais segura a resposta.
Casos concluídos cuja sobreposição fica abaixo do próprio ruído.
noise_floor - overlap: quanto das primeiras posições o elemento move além do ruído.
constraints, hard, size, exclude Para onde foram, sem o elemento, os itens com que a pessoa interagiu, nos casos concluídos.
Show child attributes
Show child attributes
not_quality, model_reaction_not_measured, trivial_for_hard, few_cases, noisy_tool, cases_skipped, dry_run overlap@k médio da base com ela mesma.
overlap@k médio da base com a variante.
O teste do sinal bilateral de below_noise contra above_noise: quão provável é uma divisão assim desigual quando o elemento não muda nada.
1 - 512Os casos não concluídos.
Show child attributes
Show child attributes

