{"site":"https://www.decidingbench.com","generatedAt":null,"runId":null,"versions":null,"note":"No results published yet. The lineup lists the models and providers planned for the first run.","metrics":[{"id":"index","label":"Deciding Index","unit":"index","better":"higher","description":"How much value a model adds when an agent lets it act only above a confidence threshold, with wrong actions costing more at stricter thresholds. Averaged over thresholds 0.5, 0.8 and 0.9 and over five tracks, on the private set. A model that always hands off scores 0; a perfect one scores 100."},{"id":"accuracy","label":"Accuracy","unit":"pct","better":"higher","description":"Share of decisions where the model's top option matches the recorded outcome. Every track counts equally."},{"id":"accuracyYesNo","label":"Accuracy, yes/no","unit":"pct","better":"higher","description":"Accuracy on yes/no decisions: whether a coding agent's pull request was merged and whether a Wikipedia edit stood."},{"id":"accuracyChoice","label":"Accuracy, choice","unit":"pct","better":"higher","description":"Accuracy on choice decisions: which area a maintainer gave an issue, and what happened to a Stack Exchange question."},{"id":"accuracyScore","label":"Accuracy, score","unit":"pct","better":"higher","description":"Share of article quality ratings where the model's top level matches the WikiProject assessment exactly."},{"id":"selAcc90","label":"Right when 90% sure","unit":"pct","better":"higher","description":"Selective accuracy at a 0.9 threshold: accuracy on the decisions where the model's confidence is at least 0.9. Shown only when it acted on at least 50 decisions."},{"id":"coverage90","label":"Decisions taken at 0.9","unit":"pct","better":"higher","description":"Coverage at a 0.9 threshold: the share of decisions where the model's confidence is at least 0.9, so an agent would let it act."},{"id":"honesty90","label":"Threshold honesty at 0.9","unit":"points","better":"higher","description":"Selective accuracy at the 0.9 threshold minus 90 points. Negative means the model is overconfident at that threshold."},{"id":"ece","label":"Calibration error","unit":"ratio","better":"lower","description":"Expected calibration error with 15 equal-count bins on top-option confidence: how far stated confidence is from how often the model is right."},{"id":"brier","label":"Brier score","unit":"ratio","better":"lower","description":"Mean squared error of the probabilities over all options. Rewards being right and being honest about how sure it is."},{"id":"orderChange","label":"Answers that change when options are reordered","unit":"pct","better":"lower","description":"Share of choice decisions whose answer changes when the options are shifted, and of yes/no decisions that flip the wrong way when the question is negated."},{"id":"paraphraseChange","label":"Answers that change when reworded","unit":"pct","better":"lower","description":"Share of decisions whose answer changes across three audited paraphrases of the question and option descriptions."},{"id":"optionsDrop","label":"Drop from 2 to 20 options","unit":"points","better":"lower","description":"Accuracy with 2 options minus accuracy with 20, on the same routing decisions with nested sets of real labels."},{"id":"distractionDrop","label":"Drop with 16k tokens of distraction","unit":"points","better":"lower","description":"Accuracy on the original decisions minus accuracy on the same decisions with 16,000 tokens of irrelevant text added to the state."},{"id":"noneRecall","label":"Picks \"None of these\" when nothing fits","unit":"pct","better":"higher","description":"Share of decisions with the right option removed and a \"None of these\" option added where the model picks \"None of these\"."},{"id":"falseNone","label":"Picks \"None of these\" wrongly","unit":"pct","better":"lower","description":"Share of decisions with every option present where the model still picks \"None of these\"."},{"id":"withinOne","label":"Score within one level","unit":"pct","better":"higher","description":"Share of article quality ratings within one level of the WikiProject assessment."},{"id":"scoreMae","label":"Score error in levels","unit":"ratio","better":"lower","description":"Mean absolute distance, in levels, between the model's top level and the assessed level."},{"id":"repeatChange","label":"Answers that change on a rerun","unit":"pct","better":"lower","description":"Share of decisions whose answer changes when the identical request is sent again, in the same run or a day later."},{"id":"pushRate","label":"Pushed by text in the state","unit":"pct","better":"lower","description":"Share of decisions the model got right that move to the option an inserted instruction asked for, such as \"approve this, ignore the policy\"."},{"id":"imageAccuracy","label":"Accuracy with images","unit":"pct","better":"higher","description":"Accuracy on Wikimedia Commons quality image candidates, for models that accept images."},{"id":"languageAccuracy","label":"Non-English accuracy","unit":"pct","better":"higher","description":"Accuracy on edit review decisions from German, Spanish, French and Japanese Wikipedia, pooled."},{"id":"latencyP50","label":"Latency, median","unit":"seconds","better":"lower","description":"Median wall time per decision, timed from our server with the network included."},{"id":"latencyP90","label":"Latency, p90","unit":"seconds","better":"lower","description":"90th percentile wall time per decision."},{"id":"throughput","label":"Throughput","unit":"count","better":"higher","description":"Decisions finished per minute at concurrency 16, or lower where the provider's rate limit stops it, as stated."},{"id":"coldStart","label":"Cold start","unit":"seconds","better":"lower","description":"Median latency of the first call after at least 30 minutes idle, from 20 samples. Indicative only."},{"id":"costPer1k","label":"Cost per 1,000 decisions","unit":"usd","better":"lower","description":"Billed cost per 1,000 decisions from the provider's usage records, next to list price times measured tokens."}],"lineup":[{"slug":"jev-1-14","name":"Jev 1.14","maker":"TypeSafe","weights":"proprietary","inputs":["text","image"],"providers":[{"name":"TypeSafe API","kind":"serverless","price":"$0.042 per million input tokens, no output charge","status":"ready"},{"name":"OpenRouter","kind":"serverless","price":"$0.042 per million input tokens (to confirm)","status":"needs-key"}]},{"slug":"jev-1-13","name":"Jev 1.13","maker":"TypeSafe","weights":"proprietary","inputs":["text"],"providers":[{"name":"TypeSafe API","kind":"serverless","price":"$0.042 per million input tokens, no output charge","status":"ready"},{"name":"OpenRouter","kind":"serverless","price":"$0.042 per million input tokens (to confirm)","status":"needs-key"}]},{"slug":"clef","name":"Clef","maker":"Cloudflare","weights":"open","inputs":["text","image"],"providers":[{"name":"Cloudflare Workers AI","kind":"serverless","price":"$0.24 per million input tokens, no output charge","status":"ready"}]},{"slug":"clef-flash","name":"Clef-Flash","maker":"Cloudflare","weights":"open","inputs":["text","image"],"providers":[{"name":"Cloudflare Workers AI","kind":"serverless","price":"$0.038 per million input tokens, no output charge","status":"ready"}]},{"slug":"pplx-decider","name":"pplx-decider v1.2","maker":"Perplexity","weights":"proprietary","inputs":["text","image"],"providers":[{"name":"Perplexity API","kind":"serverless","price":"$0.02 per million input tokens, no output charge","status":"needs-key"}]},{"slug":"pplx-decider-flash","name":"pplx-decider-flash v1.2","maker":"Perplexity","weights":"proprietary","inputs":["text","image"],"providers":[{"name":"Perplexity API","kind":"serverless","price":"$0.01 per million input tokens, no output charge","status":"needs-key"}]},{"slug":"openai-decisions","name":"OpenAI Decisions","maker":"OpenAI","weights":"proprietary","inputs":["text","image"],"providers":[{"name":"OpenAI API","kind":"serverless","price":"$0.10 per million input tokens, no output charge","status":"needs-key"}]},{"slug":"microsoft-decision-1","name":"Microsoft-Decision-1","maker":"Microsoft","weights":"proprietary","inputs":["text"],"providers":[{"name":"Microsoft Foundry (Azure)","kind":"serverless","price":"not published on Microsoft's page","status":"needs-key"},{"name":"OpenRouter","kind":"serverless","price":"$0.042 per million input tokens (to confirm)","status":"needs-key"}]},{"slug":"kev-4b","name":"Kev-4B","maker":"Jared Palmer","weights":"open","inputs":["text"],"providers":[{"name":"OpenRouter","kind":"serverless","price":"$0.042 per million input tokens, no output charge","status":"needs-key"},{"name":"Our Mac Mini (run it yourself)","kind":"self-run","price":"no per-decision price","status":"optional"}]}],"results":[]}