{ "benchmark": "JevBench", "revision": "v1.6.28", "snapshot_date": "2026-10-18", "source_url": "https://benchmarkheaven.com/api/jevbench/latest", "leaderboard_url": "https://benchmarkheaven.com/jev-models", "measurement_report": "https://benchmarkheaven.com/jev-models/v1.6.28", "scope": "Open-weights Capability ranking and selected API comparisons under official caps. Perplexity is shown explicitly as over the cost cap; Microsoft A2 is preliminary and unranked.", "capability_policy": { "definition": "Arithmetic mean of Intelligence and Calibration; ranked models within both official caps", "reference_key": "jev-1.13.0@v1.5.7", "cost_cap_usd_per_1000": 0.06459465517241379, "median_latency_cap_s": 1.2329566404223442, "factor": 2 }, "models": [ { "key": "basal-omni-1.0-ultra", "name": "basal-omni-1.0-ultra (Remek Kinas, Gemma-4-31B-it + merged LoRA, typed-decision readout, self-hosted)", "board": "open", "source_url": "https://huggingface.co/Remek/basal-omni-1.0-ultra", "axes": { "intelligence": 74.05810874808022, "calibration": 91.0178774866748, "speed": 87.31015625661291, "cost": 50.50507014318134 }, "capability": { "score": 82.53799311737751, "rank": 2, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 1, "api_board_rank": null }, "composite_score": 71.75766176892091, "rank": 8, "open_board_rank": 2, "api_board_rank": null, "measured_in": "v1.6.28", "draw": { "id": "v1.6-regular-20261012", "label": "Second regular draw", "release": "v1.6.28", "measured_on": "2026-10-17", "status": "anchor-runs-pending" }, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.04465098000000001, "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base google/gemma-4-31B-it -> google/gemma-4-31b-it USD 0.09/M input, USD 0.33999999999999997/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. USD per 1,000 decisions over the full 1,500-item denominator. Tokens as reported by the author's own server (usage.input_tokens = tokens of the rendered chat prompt incl. system prompt, options and the answer prefill; output_tokens 0, nothing is generated - the model reads the option-code logits at one answer position in one forward pass). The offline count through the author's own renderer and the pinned tokenizer predicted the server's numbers exactly, item by item: 744,183 input tokens over the 1,485 answered items." }, "latency": { "p50_s_raw": 0.10191065148683265, "p50_s_adjusted": 0.3538213029736653, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.10191065148683265, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "quyet-1-0-large", "name": "Quyet-1.0-Large (Chinh Nguyen, Gemma-4-31B decoder, option-letter logits)", "board": "open", "source_url": "https://huggingface.co/chinhnc/Quyet-1.0-Large", "axes": { "intelligence": 73.40135940607429, "calibration": 89.97624968733336, "speed": 86.87440406834797, "cost": 50.546097335562465 }, "capability": { "score": 81.68880454670384, "rank": 3, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 2, "api_board_rank": null }, "composite_score": 71.38706329141472, "rank": 11, "open_board_rank": 4, "api_board_rank": null, "measured_in": "v1.6.0", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.04451059729064039, "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact base model google/gemma-4-31B-it; listed since Apr 2026), read 2026-10-04 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged" }, "latency": { "p50_s_raw": 0.11577823059633374, "p50_s_adjusted": 0.3815564611926675, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.11577823059633374, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "jeff_1_0_large", "name": "Jeff-1.0-Large", "board": "open", "source_url": "https://huggingface.co/jgeuter/Jeff-1.0-Large", "axes": { "intelligence": 71.89886397161271, "calibration": 91.10757097649757, "speed": 88.82411852274376, "cost": 49.20157807658991 }, "capability": { "score": 81.50321747405513, "rank": 4, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 3, "api_board_rank": null }, "composite_score": 68.59329511867139, "rank": 21, "open_board_rank": 11, "api_board_rank": null, "measured_in": "v1.6.2", "draw": { "id": "v1.6-fastlane-20261009", "label": "Fast-lane draw", "release": "v1.6.4", "measured_on": "2026-10-09", "status": "anchor-runs-pending" }, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.04934926977687626, "basis": "Public exact base-model OpenRouter reference for google/gemma-4-31b-it, 2026-10-09: cheapest paid DeepInfra Turbo $0.09/M input, $0.34/M output. Jeff uses native bf16 single forward-pass decision readout with zero generated output tokens. Reference tariff is not an actual Jeff hosting bill. Primary source https://openrouter.ai/api/v1/models/google/gemma-4-31b-it/endpoints; source receipt retained." }, "latency": { "p50_s_raw": 0.08006057143211365, "p50_s_adjusted": 0.3101211428642273, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.08006057143211365, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "metask_jev_rain_31b_trained_v3", "name": "metask-jev-rain-31B-trained service v3", "board": "open", "source_url": "https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained", "axes": { "intelligence": 72.59243765084337, "calibration": 86.60202339920977, "speed": 84.66561996644248, "cost": 47.944910134589236 }, "capability": { "score": 79.59723052502656, "rank": 7, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 4, "api_board_rank": null }, "composite_score": 63.422592654952346, "rank": 39, "open_board_rank": 25, "api_board_rank": null, "measured_in": "v1.6.17", "draw": { "id": "v1.6-regular-20261012", "label": "Second regular draw", "release": "v1.6.28", "measured_on": "2026-10-13", "status": "anchor-runs-pending" }, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.05434626, "basis": "v1.5-M2 base-model reference: exact base google/gemma-4-31B-it, OpenRouter google/gemma-4-31b-it list price USD 0.09/M input, USD 0.34/M output (the same reference as the published metask_jev_rain_31b_trained row; snapshot read 2026-10-13T12:10:25Z by the order's public-docs receipt) x the service's own measured usage tokens (840,049 in / 17,397 out); estimated, not charged." }, "latency": { "p50_s_raw": 0.18078989698551595, "p50_s_adjusted": 0.5115797939710319, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.18078989698551595, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "decisio-gemma-4-31b-v080", "name": "decisio v0.8.0 on gemma-4-31B-it (frozen, FP8 on load, one prefill per state, self-hosted)", "board": "open", "source_url": "https://github.com/aminry/decisio", "axes": { "intelligence": 70.42321091828492, "calibration": 88.71627256952715, "speed": 91.18580810090454, "cost": 51.66530928015878 }, "capability": { "score": 79.56974174390604, "rank": 8, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 5, "api_board_rank": null }, "composite_score": 71.6926576652985, "rank": 9, "open_board_rank": 3, "api_board_rank": null, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.04084664184157075, "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff) and CLEAN: unlike run 54's FP8 sibling, the frozen pricing snapshot resolves this EXACT base - google/gemma-4-31B-it - to USD 0.09 per 1M input and USD 0.34 per 1M output. Read before the pod and asserted in make_registry_r55.py. x this row's own measured tokens, the deck31b method. Nothing is provisional and no release-lane line is open on it. Estimated, not charged" }, "latency": { "p50_s_raw": 0.047096215188503265, "p50_s_adjusted": 0.24419243037700653, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.047096215188503265, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "h2o-lightning-4b", "name": "H2O-Lightning-4B v1.1 (H2O.ai, Qwen3.5-4B fine-tune, stock vLLM + open shim)", "board": "open", "source_url": "https://huggingface.co/h2oai/h2o-lightning-4b", "axes": { "intelligence": 60.03839906196252, "calibration": 89.95399367673842, "speed": 92.61084474720658, "cost": 60.29882443070202 }, "capability": { "score": 74.99619636935047, "rank": 16, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": 10, "api_board_rank": null }, "composite_score": 72.5232271521435, "rank": 6, "open_board_rank": 1, "api_board_rank": null, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.021055837438423645, "basis": "ESTIMATE (self-hosted open weights, no public tariff): cost per 1,000 decisions measured for H2O-Lightning-4B v1.0 on the v1.5 pool (same base, shim and token path) carried for v1.1, the practice of every Qwen3.5-4B row on this board; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off. Cross-check: v1.1's own v1.6 input tokens on the common cost basis give USD 0.0204 per 1,000 (within 3 %)" }, "latency": { "p50_s_raw": 0.02928461297415197, "p50_s_adjusted": 0.20856922594830393, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.02928461297415197, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "jev-1.13.0", "name": "Jev 1.13.0 (TypeSafe AI)", "board": "reference", "source_url": "https://docs.typesafe.ai", "axes": { "intelligence": 63.62075784024779, "calibration": 90.63220203152369, "speed": 91.49989299046987, "cost": 54.733842199536014 }, "capability": { "score": 77.12647993588574, "rank": 13, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": null, "api_board_rank": null }, "composite_score": 71.49118639284514, "rank": 10, "open_board_rank": null, "api_board_rank": 7, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "tariff", "usd_per_1000_decisions": 0.03227542180094787, "basis": "TypeSafe list tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage" }, "latency": { "p50_s_raw": 0.23910215869545937, "p50_s_adjusted": 0.23910215869545937, "adjustment": "none (API)", "p50_s_measured": 0.23910215869545937, "latency_basis": "api-end-to-end" } }, { "key": "openai-decisions", "name": "OpenAI Decisions (gpt-6-luna)", "board": "api", "source_url": null, "axes": { "intelligence": 56.9, "calibration": 90.1, "speed": 89.25, "cost": 48.6 }, "capability": { "score": 73.5, "rank": 21, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": null, "api_board_rank": 10 }, "composite_score": 62.5, "rank": 43, "open_board_rank": null, "api_board_rank": 15, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "tariff", "usd_per_1000_decisions": 0.0517, "basis": "measured: token usage of the 6 Oct 2026 A5 u P run x tariff USD 0.1/M input, 0.0/M output (spec openai-decisions.json; OpenAI Decisions API (POST /v1/decisions, gpt-6-luna) list price USD 0.10 per 1M input tokens, no output/cache charges; source https://developers.openai.com/api/docs/guides/decisions (checked 2026-10-06); actual usage), per 1,000 decisions on the v1.6.1 common cost basis (592 of 600 items; the 8 A5 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a5.json). List price as of 6 Oct 2026 (launch day); re-checked at each revision." }, "latency": { "p50_s_raw": 0.298, "p50_s_adjusted": 0.298, "adjustment": "none (API)", "p50_s_measured": 0.298, "latency_basis": "api-end-to-end" } }, { "key": "microsoft-decision-1", "name": "Microsoft-Decision-1 (Azure Foundry)", "board": "api", "source_url": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/", "axes": { "intelligence": 57.16958510591244, "calibration": 84.39873071167106, "speed": 81.54517543213291, "cost": 61.452075716621465 }, "capability": { "score": 70.78415790879176, "rank": 32, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": null, "api_board_rank": 12 }, "composite_score": 69.11289175277149, "rank": 18, "open_board_rank": null, "api_board_rank": 10, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "tariff", "usd_per_1000_decisions": 0.0192721784989858, "basis": "Advertised Azure Foundry tariff .042/M input, output free; genuine native usage, billed invoice unknown" }, "latency": { "p50_s_raw": 0.45925578847527504, "p50_s_adjusted": 0.45925578847527504, "adjustment": "none (API)", "p50_s_measured": 0.45925578847527504, "latency_basis": "api-end-to-end" } }, { "key": "pplx-decider-v1-1-27b", "name": "Perplexity Decider v1.1 27B (Qwen3.8-27B, noncausal decision readout)", "board": "open", "source_url": "https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b", "axes": { "intelligence": 75.71677945512734, "calibration": 89.86695733940225, "speed": 86.79279341461623, "cost": 29.886617009940878 }, "capability": { "score": 82.79186839726479, "rank": null, "eligible": false, "within_caps": false, "reasons": [ "cost 6.7× Jev (v1.5 reference)" ], "open_board_rank": null, "api_board_rank": null }, "composite_score": 20.617642759667973, "rank": 111, "open_board_rank": 84, "api_board_rank": null, "measured_in": "v1.6.1", "draw": null, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.21732654028436021, "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged" }, "latency": { "p50_s_raw": 0.12236087050405331, "p50_s_adjusted": 0.39472174100810664, "adjustment": "x2 + 0.15 s (assumption, not measured)", "p50_s_measured": 0.12236087050405331, "latency_basis": "measured-loopback", "p50_s_measured_condition": "on-GPU, no network", "latency_note": "Self-hosted models are timed on our own hardware over loopback, without a network, so their measured p50 is not directly comparable with API latency, which we time end to end from our measuring host, network included. The ranking and the latency cap use the same estimate for every self-hosted row: 2 × measured p50 + 150 ms." } }, { "key": "kettei-fat", "name": "kettei-fat (Sqwish)", "board": "api", "source_url": "https://docs.sqwish.ai", "axes": { "intelligence": 79.83918785475791, "calibration": 92.78078361709031, "speed": 93.19124027169539, "cost": 57.53867836279444 }, "capability": { "score": 86.3099857359241, "rank": 1, "eligible": true, "within_caps": true, "reasons": [], "open_board_rank": null, "api_board_rank": 1 }, "composite_score": 77.80056651426555, "rank": 1, "open_board_rank": null, "api_board_rank": 1, "measured_in": "v1.6.24", "draw": { "id": "v1.6-regular-20261012", "label": "Second regular draw", "release": "v1.6.28", "measured_on": "2026-10-16", "status": "anchor-runs-pending" }, "ranked": true, "listing": "ranked", "not_ranked_because": null, "price": { "kind": "estimate", "usd_per_1000_decisions": 0.026024199999999994, "basis": "Sqwish public price list, USD 0.06/M input, output not billed (docs.sqwish.ai/guides/pricing, fetched 16 Oct 2026 16:17Z, saved copy sha256 698a29e5...): kettei fat at USD 0.06 per million input tokens for decisions; \"The models generate no output tokens, and none are billed\"; \"standard list prices, the same since launch on 14 October 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4 and Nimble), through a scoped re-freeze of the frozen pricing module that adds the two kettei keys and nothing else, with a zero-difference proof for every other registry entry. The once-per-inbox USD 5.00 starter credit is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (650,605 input tokens over the 1,485 answered items, 0 output tokens), divided by the full 1,500-row basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers, as every refused request on this draw does (d1-ultra's identical 15 refusals included). The unchanged runner also wrote a conservative cost ceiling into those 15 refused rows (65,536 input tokens each, the model's input limit); the scoring copy drops that one field on those 15 rows only. With the ceiling charged, the figure would be USD 0.065346 per 1,000 decisions (1.012x the official cost cap); that is a sensitivity, not this row's cost. Not a bill and not a quote." }, "latency": { "p50_s_raw": 0.20005768549162894, "p50_s_adjusted": 0.20005768549162894, "adjustment": "none (API)", "p50_s_measured": 0.20005768549162894, "latency_basis": "api-end-to-end" } }, { "key": "microsoft-decision-1-a2-20261012", "name": "Microsoft-Decision-1 (Azure Foundry) — A2 re-run", "board": "api", "source_url": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/", "axes": { "intelligence": 59.226098571054024, "calibration": 85.15566582345811, "speed": 88.69804989142763, "cost": 62.19876038790379 }, "capability": { "score": 72.19088219725606, "rank": null, "eligible": false, "within_caps": true, "reasons": [], "open_board_rank": null, "api_board_rank": null }, "composite_score": 71.4549280926844, "rank": null, "open_board_rank": null, "api_board_rank": null, "measured_in": "v1.6.14", "draw": { "id": "v1.6-regular-20261012", "label": "Second regular draw", "release": "v1.6.28", "measured_on": "2026-10-12", "status": "anchor-runs-pending" }, "ranked": false, "listing": "preliminary", "not_ranked_because": "A2 supplement measurement with an incomplete equating pool (four of six); excluded from every ranking.", "price": { "kind": "tariff", "usd_per_1000_decisions": 0.01819874, "basis": "LEAD DECISION r3: USD 0.042/M input and USD 0/M output, basis of the live Microsoft-Decision-1 (Azure Foundry) row on https://benchmarkheaven.com/jev-models/api. Reported usage across all 600 decisions, including seven HTTP422 refusals retained in denominator; frozen parent all-item cost basis. Invoice, warmup and refusal charges unknown." }, "latency": { "p50_s_raw": 0.3374342759998399, "p50_s_adjusted": 0.3374342759998399, "adjustment": "none (API)", "p50_s_measured": 0.3374342759998399, "latency_basis": "api-end-to-end" } } ] }