{ "benchmark": "Werdykt v1", "date": "2026-10-04", "items": 5000, "categories": 10, "h100_usd_per_hour": 3.95, "sources": { "overall": "Werdykt v1 · eksport wyników z 2026-10-04", "polish": "basal-1.5 in Polish · raport z 3 października 2026, uzupełniony o nowe systemy z eksportu 2026-10-04" }, "models": [ { "name": "Gemini 3.8 Flash", "size": "—", "kind": "api", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.9956, "ci95": [ 0.9938, 0.9972 ], "pl": 0.9956, "en": 0.9956, "categories": { "abstain": 1.0, "action": 0.996, "contract": 0.994, "judge": 0.99, "long": 0.992, "realdoc": 0.996, "retrieval": 0.996, "routing": 0.992, "rules": 1.0, "score": 1.0 }, "latency_ms": 3314.5, "p95_ms": 7379.4, "cost_per_1k_usd": 2.5297, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "Claude Opus 5.5", "size": "—", "kind": "api", "basal": false, "independent": false, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.994, "ci95": [ 0.9918, 0.9962 ], "pl": 0.992, "en": 0.996, "categories": { "abstain": 0.998, "action": 1.0, "contract": 0.992, "judge": 0.99, "long": 0.996, "realdoc": 0.99, "retrieval": 0.998, "routing": 0.978, "rules": 1.0, "score": 0.998 }, "latency_ms": 5531.3, "p95_ms": 9362.2, "cost_per_1k_usd": 15.1738, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "GPT-6.1 Sol", "size": "—", "kind": "api", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.9938, "ci95": [ 0.9916, 0.996 ], "pl": 0.9924, "en": 0.9952, "categories": { "abstain": 1.0, "action": 0.998, "contract": 0.99, "judge": 0.974, "long": 1.0, "realdoc": 0.992, "retrieval": 1.0, "routing": 0.986, "rules": 1.0, "score": 0.998 }, "latency_ms": 2213.6, "p95_ms": 4928.0, "cost_per_1k_usd": 3.178, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "GPT-6 Astra", "size": "—", "kind": "api", "basal": false, "independent": false, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.9934, "ci95": [ 0.9912, 0.9956 ], "pl": 0.992, "en": 0.9948, "categories": { "abstain": 1.0, "action": 1.0, "contract": 0.99, "judge": 0.97, "long": 1.0, "realdoc": 0.99, "retrieval": 1.0, "routing": 0.988, "rules": 1.0, "score": 0.996 }, "latency_ms": 1993.9, "p95_ms": 4356.5, "cost_per_1k_usd": 15.8178, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "GPT-6 Luna", "size": "—", "kind": "api", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.9832, "ci95": [ 0.9794, 0.9864 ], "pl": 0.9784, "en": 0.988, "categories": { "abstain": 0.996, "action": 0.992, "contract": 0.984, "judge": 0.944, "long": 0.966, "realdoc": 0.996, "retrieval": 0.99, "routing": 0.986, "rules": 0.988, "score": 0.99 }, "latency_ms": 2380.1, "p95_ms": 4210.5, "cost_per_1k_usd": 0.188, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "Gemini 3.1 Pro (preview)", "size": "—", "kind": "api", "basal": false, "independent": false, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 5, "control": false, "hardware": "API dostawcy", "macro": 0.9788, "ci95": [ 0.9748, 0.9824 ], "pl": 0.9792, "en": 0.9784, "categories": { "abstain": 0.99, "action": 0.996, "contract": 0.984, "judge": 0.98, "long": 0.872, "realdoc": 0.994, "retrieval": 0.998, "routing": 0.984, "rules": 0.996, "score": 0.994 }, "latency_ms": 5539.5, "p95_ms": 19820.1, "cost_per_1k_usd": 12.2981, "engine": "API", "reasoning": "Domyślne dostawcy" }, { "name": "Surogate Rune 26B-A4B v3", "size": "26B-A4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × RTX PRO 6000 Blackwell 96 GB", "macro": 0.8168, "ci95": [ 0.8066, 0.8264 ], "pl": 0.8088, "en": 0.8248, "categories": { "abstain": 0.78, "action": 0.85, "contract": 0.772, "judge": 0.878, "long": 0.754, "realdoc": 0.98, "retrieval": 0.982, "routing": 0.976, "rules": 0.41, "score": 0.786 }, "latency_ms": 164.1, "p95_ms": 1509.9, "cost_per_1k_usd": 0.15328, "engine": "surogate · C++", "reasoning": "—" }, { "name": "Jev 1.13.0", "size": "—", "kind": "api", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "API dostawcy", "macro": 0.8164, "ci95": [ 0.807, 0.8258 ], "pl": 0.8204, "en": 0.8124, "categories": { "abstain": 0.74, "action": 0.828, "contract": 0.756, "judge": 0.906, "long": 0.726, "realdoc": 0.994, "retrieval": 0.99, "routing": 0.984, "rules": 0.442, "score": 0.798 }, "latency_ms": 286.4, "p95_ms": 378.8, "cost_per_1k_usd": 0.071007, "engine": "API", "reasoning": "—" }, { "name": "Cygnet", "size": "12B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7852, "ci95": [ 0.7752, 0.7954 ], "pl": 0.7904, "en": 0.78, "categories": { "abstain": 0.69, "action": 0.85, "contract": 0.74, "judge": 0.874, "long": 0.644, "realdoc": 0.97, "retrieval": 0.988, "routing": 0.964, "rules": 0.376, "score": 0.756 }, "latency_ms": 39.6, "p95_ms": 276.9, "cost_per_1k_usd": 0.05218, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Winnow-12B (Q8)", "size": "12B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.785, "ci95": [ 0.7742, 0.7944 ], "pl": 0.7856, "en": 0.7844, "categories": { "abstain": 0.7, "action": 0.846, "contract": 0.716, "judge": 0.88, "long": 0.642, "realdoc": 0.968, "retrieval": 0.978, "routing": 0.978, "rules": 0.362, "score": 0.78 }, "latency_ms": 126.2, "p95_ms": 1096.7, "cost_per_1k_usd": 0.25985, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "basal-1.5-max", "size": "11B", "kind": "open", "basal": true, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7728, "ci95": [ 0.7624, 0.7828 ], "pl": 0.7768, "en": 0.7688, "categories": { "abstain": 0.734, "action": 0.734, "contract": 0.71, "judge": 0.828, "long": 0.682, "realdoc": 0.972, "retrieval": 0.978, "routing": 0.982, "rules": 0.346, "score": 0.762 }, "latency_ms": 67.1, "p95_ms": 476.6, "cost_per_1k_usd": 0.10808, "engine": "SGLang", "reasoning": "—" }, { "name": "Jev-Omni", "size": "12B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7534, "ci95": [ 0.7422, 0.7638 ], "pl": 0.7632, "en": 0.7436, "categories": { "abstain": 0.664, "action": 0.746, "contract": 0.716, "judge": 0.822, "long": 0.622, "realdoc": 0.958, "retrieval": 0.966, "routing": 0.932, "rules": 0.368, "score": 0.74 }, "latency_ms": 1131.5, "p95_ms": 1555.0, "cost_per_1k_usd": 1.23966, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "basal-1.5", "size": "4.5B", "kind": "open", "basal": true, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7212, "ci95": [ 0.7102, 0.7326 ], "pl": 0.7244, "en": 0.718, "categories": { "abstain": 0.674, "action": 0.63, "contract": 0.654, "judge": 0.784, "long": 0.612, "realdoc": 0.926, "retrieval": 0.956, "routing": 0.952, "rules": 0.344, "score": 0.68 }, "latency_ms": 33.8, "p95_ms": 270.2, "cost_per_1k_usd": 0.04773, "engine": "SGLang", "reasoning": "—" }, { "name": "Decision-2.0-Nox-4B", "size": "4B (4.21B)", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 NVL 94 GB", "macro": 0.7164, "ci95": [ 0.7046, 0.728 ], "pl": 0.7092, "en": 0.7236, "categories": { "abstain": 0.624, "action": 0.7, "contract": 0.678, "judge": 0.686, "long": 0.622, "realdoc": 0.938, "retrieval": 0.948, "routing": 0.948, "rules": 0.33, "score": 0.69 }, "latency_ms": 863.8, "p95_ms": 1302.4, "cost_per_1k_usd": 1.00931, "engine": "eager · H100 NVL", "reasoning": "—" }, { "name": "Clef-Flash", "size": "9B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7158, "ci95": [ 0.704, 0.727 ], "pl": 0.7008, "en": 0.7308, "categories": { "abstain": 0.616, "action": 0.688, "contract": 0.66, "judge": 0.718, "long": 0.66, "realdoc": 0.954, "retrieval": 0.928, "routing": 0.928, "rules": 0.32, "score": 0.686 }, "latency_ms": 1700.2, "p95_ms": 1857.0, "cost_per_1k_usd": 1.89897, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Qwen3-32B, zero-shot (jqv)", "size": "32B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7054, "ci95": [ 0.6942, 0.7168 ], "pl": 0.6952, "en": 0.7156, "categories": { "abstain": 0.632, "action": 0.668, "contract": 0.634, "judge": 0.77, "long": 0.546, "realdoc": 0.948, "retrieval": 0.924, "routing": 0.932, "rules": 0.34, "score": 0.66 }, "latency_ms": 129.5, "p95_ms": 910.7, "cost_per_1k_usd": 0.22978, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Hopper", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7028, "ci95": [ 0.6916, 0.7142 ], "pl": 0.694, "en": 0.7116, "categories": { "abstain": 0.566, "action": 0.652, "contract": 0.66, "judge": 0.72, "long": 0.626, "realdoc": 0.928, "retrieval": 0.946, "routing": 0.954, "rules": 0.332, "score": 0.644 }, "latency_ms": 60.4, "p95_ms": 165.4, "cost_per_1k_usd": 0.13764, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Decision 4B v1.2", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7016, "ci95": [ 0.6908, 0.7134 ], "pl": 0.6984, "en": 0.7048, "categories": { "abstain": 0.702, "action": 0.622, "contract": 0.612, "judge": 0.732, "long": 0.558, "realdoc": 0.91, "retrieval": 0.94, "routing": 0.926, "rules": 0.334, "score": 0.68 }, "latency_ms": 28.7, "p95_ms": 206.9, "cost_per_1k_usd": 0.07101, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "decider-4b v2", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.7, "ci95": [ 0.6886, 0.7114 ], "pl": 0.6968, "en": 0.7032, "categories": { "abstain": 0.602, "action": 0.658, "contract": 0.602, "judge": 0.682, "long": 0.648, "realdoc": 0.944, "retrieval": 0.924, "routing": 0.934, "rules": 0.32, "score": 0.686 }, "latency_ms": 27.8, "p95_ms": 283.3, "cost_per_1k_usd": 0.05146, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "djev", "size": "26B (4B active)", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 274, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.698, "ci95": [ 0.6872, 0.7098 ], "pl": 0.7016, "en": 0.6944, "categories": { "abstain": 0.572, "action": 0.714, "contract": 0.658, "judge": 0.846, "long": 0.306, "realdoc": 0.912, "retrieval": 0.984, "routing": 0.936, "rules": 0.346, "score": 0.706 }, "latency_ms": 66.6, "p95_ms": 159.1, "cost_per_1k_usd": 0.05256, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "spark-s1-4b-v6", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6938, "ci95": [ 0.6816, 0.7058 ], "pl": 0.6884, "en": 0.6992, "categories": { "abstain": 0.618, "action": 0.618, "contract": 0.668, "judge": 0.716, "long": 0.51, "realdoc": 0.908, "retrieval": 0.954, "routing": 0.926, "rules": 0.338, "score": 0.682 }, "latency_ms": 217.4, "p95_ms": 565.2, "cost_per_1k_usd": 0.28788, "engine": "open_spark_jev · HF", "reasoning": "—" }, { "name": "Plumb-4B", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6904, "ci95": [ 0.6788, 0.7032 ], "pl": 0.6892, "en": 0.6916, "categories": { "abstain": 0.622, "action": 0.592, "contract": 0.634, "judge": 0.722, "long": 0.586, "realdoc": 0.93, "retrieval": 0.942, "routing": 0.916, "rules": 0.318, "score": 0.642 }, "latency_ms": 27.7, "p95_ms": 245.7, "cost_per_1k_usd": 0.05525, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Decision 4B v1.1", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6902, "ci95": [ 0.6788, 0.7022 ], "pl": 0.6876, "en": 0.6928, "categories": { "abstain": 0.704, "action": 0.602, "contract": 0.598, "judge": 0.73, "long": 0.548, "realdoc": 0.904, "retrieval": 0.942, "routing": 0.914, "rules": 0.284, "score": 0.676 }, "latency_ms": 28.5, "p95_ms": 207.1, "cost_per_1k_usd": 0.12362, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "JevK5 v0.3", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6886, "ci95": [ 0.6768, 0.7012 ], "pl": 0.6828, "en": 0.6944, "categories": { "abstain": 0.592, "action": 0.616, "contract": 0.636, "judge": 0.698, "long": 0.584, "realdoc": 0.93, "retrieval": 0.944, "routing": 0.922, "rules": 0.298, "score": 0.666 }, "latency_ms": 28.4, "p95_ms": 258.5, "cost_per_1k_usd": 0.05698, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Kev 1.0 (kev-4b)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6836, "ci95": [ 0.6714, 0.6964 ], "pl": 0.6716, "en": 0.6956, "categories": { "abstain": 0.614, "action": 0.636, "contract": 0.63, "judge": 0.64, "long": 0.594, "realdoc": 0.9, "retrieval": 0.876, "routing": 0.924, "rules": 0.35, "score": 0.672 }, "latency_ms": 34.1, "p95_ms": 308.9, "cost_per_1k_usd": 0.04423, "engine": "kev.serve", "reasoning": "—" }, { "name": "Imajev-4B", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 436, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6816, "ci95": [ 0.671, 0.6924 ], "pl": 0.6656, "en": 0.6976, "categories": { "abstain": 0.762, "action": 0.68, "contract": 0.648, "judge": 0.728, "long": 0.112, "realdoc": 0.94, "retrieval": 0.944, "routing": 0.936, "rules": 0.348, "score": 0.718 }, "latency_ms": 112.7, "p95_ms": 132.3, "cost_per_1k_usd": 0.13495, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "JevK5 v0.2 (4B)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6672, "ci95": [ 0.6556, 0.6794 ], "pl": 0.664, "en": 0.6704, "categories": { "abstain": 0.588, "action": 0.58, "contract": 0.604, "judge": 0.704, "long": 0.554, "realdoc": 0.918, "retrieval": 0.894, "routing": 0.922, "rules": 0.292, "score": 0.616 }, "latency_ms": 30.7, "p95_ms": 240.4, "cost_per_1k_usd": 0.05624, "engine": "jevk5-serve", "reasoning": "—" }, { "name": "basal-1.0 · 4.5B", "size": "4.5B", "kind": "open", "basal": true, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6604, "ci95": [ 0.647, 0.6718 ], "pl": 0.662, "en": 0.6588, "categories": { "abstain": 0.584, "action": 0.606, "contract": 0.598, "judge": 0.604, "long": 0.588, "realdoc": 0.89, "retrieval": 0.912, "routing": 0.88, "rules": 0.34, "score": 0.602 }, "latency_ms": 26.5, "p95_ms": 779.5, "cost_per_1k_usd": 0.13534, "engine": "basal · fast", "reasoning": "—" }, { "name": "Malkuth-4B", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 108, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6336, "ci95": [ 0.621, 0.6466 ], "pl": 0.6224, "en": 0.6448, "categories": { "abstain": 0.49, "action": 0.6, "contract": 0.602, "judge": 0.574, "long": 0.504, "realdoc": 0.908, "retrieval": 0.844, "routing": 0.9, "rules": 0.284, "score": 0.63 }, "latency_ms": 153.2, "p95_ms": 285.2, "cost_per_1k_usd": 0.17219, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "Manchego v2.1", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6304, "ci95": [ 0.6176, 0.643 ], "pl": 0.622, "en": 0.6388, "categories": { "abstain": 0.496, "action": 0.594, "contract": 0.586, "judge": 0.708, "long": 0.452, "realdoc": 0.822, "retrieval": 0.81, "routing": 0.89, "rules": 0.348, "score": 0.598 }, "latency_ms": 73.5, "p95_ms": 400.8, "cost_per_1k_usd": 0.1149, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "local-jev (Qwen3.5-4B)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6298, "ci95": [ 0.6176, 0.6416 ], "pl": 0.6248, "en": 0.6348, "categories": { "abstain": 0.516, "action": 0.574, "contract": 0.6, "judge": 0.606, "long": 0.48, "realdoc": 0.878, "retrieval": 0.864, "routing": 0.892, "rules": 0.29, "score": 0.598 }, "latency_ms": 142.7, "p95_ms": 535.8, "cost_per_1k_usd": 0.25335, "engine": "local-jev · HF", "reasoning": "—" }, { "name": "typecastlm v1.3.0 (Qwen3.5 3.8B)", "size": "3.8B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6288, "ci95": [ 0.6172, 0.6414 ], "pl": 0.6236, "en": 0.634, "categories": { "abstain": 0.51, "action": 0.494, "contract": 0.602, "judge": 0.646, "long": 0.506, "realdoc": 0.882, "retrieval": 0.922, "routing": 0.874, "rules": 0.248, "score": 0.604 }, "latency_ms": 95.2, "p95_ms": 217.3, "cost_per_1k_usd": 0.13805, "engine": "typecastlm · local", "reasoning": "—" }, { "name": "lev", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6286, "ci95": [ 0.6164, 0.6412 ], "pl": 0.6168, "en": 0.6404, "categories": { "abstain": 0.52, "action": 0.58, "contract": 0.57, "judge": 0.598, "long": 0.56, "realdoc": 0.892, "retrieval": 0.85, "routing": 0.834, "rules": 0.284, "score": 0.598 }, "latency_ms": 157.6, "p95_ms": 538.1, "cost_per_1k_usd": 0.20125, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "jev-local (Qwen3.5-9B, HF scorer)", "size": "9B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 436, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6218, "ci95": [ 0.6108, 0.6332 ], "pl": 0.6156, "en": 0.628, "categories": { "abstain": 0.534, "action": 0.6, "contract": 0.62, "judge": 0.708, "long": 0.106, "realdoc": 0.91, "retrieval": 0.884, "routing": 0.908, "rules": 0.31, "score": 0.638 }, "latency_ms": 460.7, "p95_ms": 829.0, "cost_per_1k_usd": 2.04401, "engine": "jevlocal · HF scorer", "reasoning": "—" }, { "name": "Qwen3-4B-Instruct-2507 (raw, letter logits)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": true, "hardware": "1 × H100 80 GB", "macro": 0.6208, "ci95": [ 0.609, 0.6322 ], "pl": 0.614, "en": 0.6276, "categories": { "abstain": 0.514, "action": 0.542, "contract": 0.548, "judge": 0.63, "long": 0.522, "realdoc": 0.886, "retrieval": 0.85, "routing": 0.868, "rules": 0.256, "score": 0.592 }, "latency_ms": 40.0, "p95_ms": 161.2, "cost_per_1k_usd": 0.02217, "engine": "vLLM · logity liter", "reasoning": "—" }, { "name": "metask-jev-4b", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 441, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6116, "ci95": [ 0.599, 0.6234 ], "pl": 0.6068, "en": 0.6164, "categories": { "abstain": 0.552, "action": 0.608, "contract": 0.618, "judge": 0.66, "long": 0.092, "realdoc": 0.86, "retrieval": 0.906, "routing": 0.888, "rules": 0.31, "score": 0.622 }, "latency_ms": 107.8, "p95_ms": 177.1, "cost_per_1k_usd": 0.34, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "basal-1.5-mini", "size": "1.5B", "kind": "open", "basal": true, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6066, "ci95": [ 0.594, 0.6192 ], "pl": 0.6192, "en": 0.594, "categories": { "abstain": 0.494, "action": 0.414, "contract": 0.544, "judge": 0.666, "long": 0.536, "realdoc": 0.832, "retrieval": 0.862, "routing": 0.876, "rules": 0.304, "score": 0.538 }, "latency_ms": 14.2, "p95_ms": 346.4, "cost_per_1k_usd": 0.0535, "engine": "basal · fast", "reasoning": "—" }, { "name": "open-alternative-jev (so1, Qwen3.5-4B)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.6024, "ci95": [ 0.5894, 0.615 ], "pl": 0.5916, "en": 0.6132, "categories": { "abstain": 0.488, "action": 0.52, "contract": 0.554, "judge": 0.564, "long": 0.472, "realdoc": 0.88, "retrieval": 0.842, "routing": 0.866, "rules": 0.272, "score": 0.566 }, "latency_ms": 101.4, "p95_ms": 433.4, "cost_per_1k_usd": 0.1579, "engine": "so1 · HF + adapter HTTP", "reasoning": "—" }, { "name": "SemIf", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 436, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.588, "ci95": [ 0.5762, 0.5992 ], "pl": 0.5752, "en": 0.6008, "categories": { "abstain": 0.502, "action": 0.568, "contract": 0.592, "judge": 0.608, "long": 0.086, "realdoc": 0.886, "retrieval": 0.87, "routing": 0.888, "rules": 0.28, "score": 0.6 }, "latency_ms": 66.0, "p95_ms": 119.4, "cost_per_1k_usd": 0.08004, "engine": "Serwer modelu", "reasoning": "—" }, { "name": "decider-2b (v11)", "size": "2B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.5866, "ci95": [ 0.574, 0.5994 ], "pl": 0.5804, "en": 0.5928, "categories": { "abstain": 0.454, "action": 0.508, "contract": 0.506, "judge": 0.564, "long": 0.552, "realdoc": 0.854, "retrieval": 0.788, "routing": 0.832, "rules": 0.302, "score": 0.506 }, "latency_ms": 20.3, "p95_ms": 168.5, "cost_per_1k_usd": 0.02476, "engine": "decider-ai 1.8.1", "reasoning": "—" }, { "name": "JEV-Qwen3.5-9B-Base-NVFP4", "size": "9B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × RTX PRO 6000 Blackwell 96 GB", "macro": 0.5726, "ci95": [ 0.5592, 0.5852 ], "pl": 0.5776, "en": 0.5676, "categories": { "abstain": 0.504, "action": 0.486, "contract": 0.5, "judge": 0.608, "long": 0.47, "realdoc": 0.818, "retrieval": 0.73, "routing": 0.764, "rules": 0.296, "score": 0.55 }, "latency_ms": 31.8, "p95_ms": 177.9, "cost_per_1k_usd": 0.02636, "engine": "SGLang · NVFP4 + adapter HTTP", "reasoning": "—" }, { "name": "Phi-4-mini-instruct (raw, letter logits)", "size": "3.8B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": true, "hardware": "1 × H100 80 GB", "macro": 0.5578, "ci95": [ 0.544, 0.5702 ], "pl": 0.5368, "en": 0.5788, "categories": { "abstain": 0.478, "action": 0.45, "contract": 0.514, "judge": 0.538, "long": 0.478, "realdoc": 0.842, "retrieval": 0.758, "routing": 0.786, "rules": 0.256, "score": 0.478 }, "latency_ms": 35.9, "p95_ms": 132.9, "cost_per_1k_usd": 0.01709, "engine": "vLLM · logity liter", "reasoning": "—" }, { "name": "Decision 2B (preview)", "size": "2B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 431, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.5462, "ci95": [ 0.5336, 0.5578 ], "pl": 0.4924, "en": 0.6, "categories": { "abstain": 0.478, "action": 0.496, "contract": 0.542, "judge": 0.628, "long": 0.086, "realdoc": 0.796, "retrieval": 0.806, "routing": 0.81, "rules": 0.262, "score": 0.558 }, "latency_ms": 48.7, "p95_ms": 76.8, "cost_per_1k_usd": 0.05801, "engine": "model.decide + adapter HTTP", "reasoning": "—" }, { "name": "basal-1.0 · 1.5B", "size": "1.5B", "kind": "open", "basal": true, "independent": true, "deviation": false, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.5224, "ci95": [ 0.5094, 0.5362 ], "pl": 0.5272, "en": 0.5176, "categories": { "abstain": 0.432, "action": 0.404, "contract": 0.478, "judge": 0.446, "long": 0.458, "realdoc": 0.78, "retrieval": 0.696, "routing": 0.756, "rules": 0.302, "score": 0.472 }, "latency_ms": 14.6, "p95_ms": 354.4, "cost_per_1k_usd": 0.05485, "engine": "basal · fast", "reasoning": "—" }, { "name": "Nemotron Diffusion 8B (Nemotron_Jev)", "size": "8B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.5154, "ci95": [ 0.5016, 0.5294 ], "pl": 0.4872, "en": 0.5436, "categories": { "abstain": 0.384, "action": 0.454, "contract": 0.508, "judge": 0.54, "long": 0.37, "realdoc": 0.802, "retrieval": 0.574, "routing": 0.806, "rules": 0.248, "score": 0.468 }, "latency_ms": 38.6, "p95_ms": 210.7, "cost_per_1k_usd": 0.06283, "engine": "Nemotron_Jev · vLLM", "reasoning": "—" }, { "name": "Malkuth-2B", "size": "2B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": true, "errors": 108, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.4998, "ci95": [ 0.4874, 0.5126 ], "pl": 0.4816, "en": 0.518, "categories": { "abstain": 0.36, "action": 0.426, "contract": 0.444, "judge": 0.476, "long": 0.426, "realdoc": 0.83, "retrieval": 0.612, "routing": 0.744, "rules": 0.238, "score": 0.442 }, "latency_ms": 73.2, "p95_ms": 119.9, "cost_per_1k_usd": 0.07645, "engine": "kev.serve", "reasoning": "—" }, { "name": "Qwen3-Reranker-4B (as a decider)", "size": "4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": true, "hardware": "1 × H100 80 GB", "macro": 0.472, "ci95": [ 0.4602, 0.4844 ], "pl": 0.4592, "en": 0.4848, "categories": { "abstain": 0.324, "action": 0.42, "contract": 0.43, "judge": 0.478, "long": 0.38, "realdoc": 0.71, "retrieval": 0.612, "routing": 0.71, "rules": 0.242, "score": 0.414 }, "latency_ms": 131.6, "p95_ms": 1089.1, "cost_per_1k_usd": 0.27426, "engine": "Transformers · adapter rerankera", "reasoning": "—" }, { "name": "Certo v1", "size": "0.4B", "kind": "open", "basal": false, "independent": true, "deviation": true, "failed_calls": false, "errors": 0, "unparsed": 0, "control": false, "hardware": "1 × H100 80 GB", "macro": 0.2826, "ci95": [ 0.2704, 0.2938 ], "pl": 0.294, "en": 0.2712, "categories": { "abstain": 0.18, "action": 0.184, "contract": 0.304, "judge": 0.442, "long": 0.26, "realdoc": 0.306, "retrieval": 0.516, "routing": 0.186, "rules": 0.24, "score": 0.208 }, "latency_ms": 20.8, "p95_ms": 25.0, "cost_per_1k_usd": 0.0238, "engine": "certo · adapter HTTP", "reasoning": "—" } ], "polish": [ { "rank": 1, "name": "Jev 1.13.0", "size": "–", "kind": "api", "basal": false, "score": 0.821, "latency": "286 ms", "hardware": "API dostawcy", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 2, "name": "Surogate Rune 26B-A4B v3", "size": "26B-A4B", "kind": "open", "basal": false, "score": 0.8088, "latency": "164,1 ms", "hardware": "1 × RTX PRO 6000 Blackwell 96 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 3, "name": "Cygnet", "size": "12B", "kind": "open", "basal": false, "score": 0.79, "latency": "40 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 4, "name": "Winnow-12B (Q8)", "size": "12B", "kind": "open", "basal": false, "score": 0.785, "latency": "126 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 5, "name": "basal-1.5-max", "size": "11B", "kind": "open", "basal": true, "score": 0.777, "latency": "67 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 6, "name": "Jev-Omni", "size": "12B", "kind": "open", "basal": false, "score": 0.763, "latency": "1,132 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 7, "name": "basal-1.5", "size": "4.5B", "kind": "open", "basal": true, "score": 0.724, "latency": "34 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 8, "name": "Decision-2.0-Nox-4B", "size": "4B (4.21B)", "kind": "open", "basal": false, "score": 0.7092, "latency": "863,8 ms", "hardware": "1 × H100 NVL 94 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 9, "name": "djev", "size": "–", "kind": "open", "basal": false, "score": 0.702, "latency": "67 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 274, "unparsed": 0 }, { "rank": 10, "name": "Clef-Flash", "size": "9B", "kind": "open", "basal": false, "score": 0.701, "latency": "1,700 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 11, "name": "Decision 4B v1.2", "size": "4B", "kind": "open", "basal": false, "score": 0.699, "latency": "29 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 12, "name": "decider-4b v2", "size": "4B", "kind": "open", "basal": false, "score": 0.697, "latency": "28 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 13, "name": "Qwen3-32B, zero-shot (jqv)", "size": "32B", "kind": "open", "basal": false, "score": 0.695, "latency": "130 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 14, "name": "Hopper", "size": "4B", "kind": "open", "basal": false, "score": 0.694, "latency": "60 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 15, "name": "Plumb-4B", "size": "4B", "kind": "open", "basal": false, "score": 0.69, "latency": "28 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 16, "name": "spark-s1-4b-v6", "size": "4B", "kind": "open", "basal": false, "score": 0.6884, "latency": "217,4 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 17, "name": "Decision 4B v1.1", "size": "4B", "kind": "open", "basal": false, "score": 0.688, "latency": "29 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 18, "name": "JevK5 v0.3", "size": "4B", "kind": "open", "basal": false, "score": 0.683, "latency": "28 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 19, "name": "Kev 1.0 (kev-4b)", "size": "4B", "kind": "open", "basal": false, "score": 0.6716, "latency": "34,1 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 20, "name": "Imajev-4B", "size": "4B", "kind": "open", "basal": false, "score": 0.666, "latency": "113 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 436, "unparsed": 0 }, { "rank": 21, "name": "JevK5 v0.2 (4B)", "size": "4B", "kind": "open", "basal": false, "score": 0.664, "latency": "30,7 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 22, "name": "basal-1.0 · 4.5B", "size": "4.5B", "kind": "open", "basal": true, "score": 0.663, "latency": "27 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 23, "name": "local-jev (Qwen3.5-4B)", "size": "4B", "kind": "open", "basal": false, "score": 0.6248, "latency": "142,7 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 24, "name": "typecastlm v1.3.0 (Qwen3.5 3.8B)", "size": "3.8B", "kind": "open", "basal": false, "score": 0.6236, "latency": "95,2 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 25, "name": "Malkuth-4B", "size": "4B", "kind": "open", "basal": false, "score": 0.623, "latency": "153 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 108, "unparsed": 0 }, { "rank": 26, "name": "Manchego v2.1", "size": "–", "kind": "open", "basal": false, "score": 0.622, "latency": "74 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 27, "name": "basal-1.5-mini", "size": "1.5B", "kind": "open", "basal": true, "score": 0.619, "latency": "14 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 28, "name": "lev", "size": "–", "kind": "open", "basal": false, "score": 0.617, "latency": "158 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 29, "name": "jev-local (Qwen3.5-9B, HF scorer)", "size": "9B", "kind": "open", "basal": false, "score": 0.6156, "latency": "460,7 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 436, "unparsed": 0 }, { "rank": 30, "name": "Qwen3-4B-Instruct-2507 (raw, letter logits)", "size": "4B", "kind": "open", "basal": false, "score": 0.614, "latency": "40,0 ms", "hardware": "1 × H100 80 GB", "control": true, "errors": 0, "unparsed": 0 }, { "rank": 31, "name": "metask-jev-4b", "size": "4B", "kind": "open", "basal": false, "score": 0.607, "latency": "108 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 441, "unparsed": 0 }, { "rank": 32, "name": "open-alternative-jev (so1, Qwen3.5-4B)", "size": "4B", "kind": "open", "basal": false, "score": 0.5916, "latency": "101,4 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 33, "name": "decider-2b (v11)", "size": "2B", "kind": "open", "basal": false, "score": 0.5804, "latency": "20,3 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 34, "name": "JEV-Qwen3.5-9B-Base-NVFP4", "size": "9B", "kind": "open", "basal": false, "score": 0.5776, "latency": "31,8 ms", "hardware": "1 × RTX PRO 6000 Blackwell 96 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 35, "name": "SemIf", "size": "4B", "kind": "open", "basal": false, "score": 0.576, "latency": "66 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 436, "unparsed": 0 }, { "rank": 36, "name": "Phi-4-mini-instruct (raw, letter logits)", "size": "3.8B", "kind": "open", "basal": false, "score": 0.5368, "latency": "35,9 ms", "hardware": "1 × H100 80 GB", "control": true, "errors": 0, "unparsed": 0 }, { "rank": 37, "name": "basal-1.0 · 1.5B", "size": "1.5B", "kind": "open", "basal": true, "score": 0.527, "latency": "15 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 38, "name": "Decision 2B (preview)", "size": "2B", "kind": "open", "basal": false, "score": 0.4924, "latency": "48,7 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 431, "unparsed": 0 }, { "rank": 39, "name": "Nemotron Diffusion 8B (Nemotron_Jev)", "size": "8B", "kind": "open", "basal": false, "score": 0.4872, "latency": "38,6 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 }, { "rank": 40, "name": "Malkuth-2B", "size": "2B", "kind": "open", "basal": false, "score": 0.4816, "latency": "73,2 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 108, "unparsed": 0 }, { "rank": 41, "name": "Qwen3-Reranker-4B (as a decider)", "size": "4B", "kind": "open", "basal": false, "score": 0.4592, "latency": "131,6 ms", "hardware": "1 × H100 80 GB", "control": true, "errors": 0, "unparsed": 0 }, { "rank": 42, "name": "Certo v1", "size": "0.4B", "kind": "open", "basal": false, "score": 0.294, "latency": "20,8 ms", "hardware": "1 × H100 80 GB", "control": false, "errors": 0, "unparsed": 0 } ], "frontier": [ { "name": "Gemini 3.8 Flash", "score": 0.996, "latency": "3.3 s", "independent": true }, { "name": "GPT-6.1 Sol", "score": 0.992, "latency": "2.2 s", "independent": true }, { "name": "Claude Opus 5.5", "score": 0.992, "latency": "5.5 s", "independent": false }, { "name": "GPT-6 Astra", "score": 0.992, "latency": "2.0 s", "independent": false }, { "name": "Gemini 3.1 Pro (preview)", "score": 0.979, "latency": "5.5 s", "independent": false }, { "name": "GPT-6 Luna", "score": 0.978, "latency": "2.4 s", "independent": true } ], "supplementary": [ { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "fast", "precision": null, "orders": 1, "subset": "full", "items": 5000, "macro": 0.7202, "ci95": null, "pl": null, "en": null, "latency_ms": 32.0, "p95_ms": 331.3, "cost_per_1k_usd": 0.09693 }, { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "vLLM", "precision": "FP8", "orders": 2, "subset": "full", "items": 5000, "macro": 0.7168, "ci95": null, "pl": null, "en": null, "latency_ms": 35.6, "p95_ms": 232.5, "cost_per_1k_usd": 0.04312 }, { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "vLLM", "precision": "FP8", "orders": 1, "subset": "s2000", "items": 2000, "macro": 0.718, "ci95": null, "pl": null, "en": null, "latency_ms": 29.2, "p95_ms": 205.1, "cost_per_1k_usd": 0.0416 }, { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "fast", "precision": null, "orders": 2, "subset": "s2000", "items": 2000, "macro": 0.723, "ci95": null, "pl": null, "en": null, "latency_ms": 36.0, "p95_ms": 1116.6, "cost_per_1k_usd": 0.19855 }, { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "vLLM", "precision": "BF16", "orders": 2, "subset": "s2000", "items": 2000, "macro": 0.725, "ci95": null, "pl": null, "en": null, "latency_ms": 40.9, "p95_ms": 264.8, "cost_per_1k_usd": 0.05827 }, { "name": "basal-1.5", "size": "4.5B", "basal": true, "engine": "vLLM", "precision": "BF16", "orders": 1, "subset": "s2000", "items": 2000, "macro": 0.7235, "ci95": null, "pl": null, "en": null, "latency_ms": 33.5, "p95_ms": 274.6, "cost_per_1k_usd": 0.05324 } ], "category_models": [ "Jev 1.13.0", "Surogate Rune 26B-A4B v3", "Cygnet · 12B", "Winnow Q8 · 12B", "basal-1.5-max · 11B", "Jev-Omni · 12B", "basal-1.5 · 4.5B", "Decision-2.0-Nox-4B", "Clef-Flash · 9B", "Decision 4B v1.2", "spark-s1-4b-v6", "Kev 1.0 (kev-4b)", "JevK5 v0.2 (4B)", "basal-1.0 · 4.5B", "local-jev (Qwen3.5-4B)", "typecastlm v1.3.0 (Qwen3.5 3.8B)", "basal-1.5-mini · 1.5B", "jev-local (Qwen3.5-9B, HF scorer)", "Qwen3-4B-Instruct-2507 (raw, letter logits)", "open-alternative-jev (so1, Qwen3.5-4B)", "decider-2b (v11)", "JEV-Qwen3.5-9B-Base-NVFP4", "Phi-4-mini-instruct (raw, letter logits)", "basal-1.0 · 1.5B", "Decision 2B (preview)", "Nemotron Diffusion 8B (Nemotron_Jev)", "Malkuth-2B", "Qwen3-Reranker-4B (as a decider)", "Certo v1" ], "category_rows": [ { "key": "abstain", "name": "Brak wystarczających danych", "description": "Wybór odpowiedzi lub stwierdzenie, że nie da się jej ustalić.", "scores": [ 0.731, 0.768, 0.703, 0.707, 0.736, 0.679, 0.7, 0.632, 0.594, 0.699, 0.6, 0.596, 0.576, 0.594, 0.532, 0.484, 0.52, 0.52, 0.512, 0.512, 0.48, 0.512, 0.452, 0.416, 0.42, 0.348, 0.364, 0.288, 0.212 ] }, { "key": "action", "name": "Następny krok agenta", "description": "Działanie zgodne z podaną polityką i stanem procesu.", "scores": [ 0.836, 0.852, 0.836, 0.824, 0.728, 0.764, 0.624, 0.7, 0.664, 0.612, 0.604, 0.656, 0.6, 0.628, 0.608, 0.524, 0.42, 0.6, 0.556, 0.52, 0.516, 0.508, 0.464, 0.456, 0.456, 0.408, 0.436, 0.436, 0.192 ] }, { "key": "contract", "name": "Umowy", "description": "Interpretacja zapisów oraz decyzje na podstawie warunków umowy.", "scores": [ 0.76, 0.76, 0.752, 0.716, 0.696, 0.696, 0.648, 0.656, 0.648, 0.604, 0.648, 0.596, 0.608, 0.592, 0.588, 0.58, 0.548, 0.604, 0.544, 0.5, 0.5, 0.504, 0.496, 0.484, 0.48, 0.512, 0.436, 0.412, 0.328 ] }, { "key": "judge", "name": "Ocena odpowiedzi", "description": "Poprawność odpowiedzi i porównywanie dwóch propozycji.", "scores": [ 0.92, 0.868, 0.892, 0.888, 0.856, 0.844, 0.8, 0.672, 0.7, 0.696, 0.696, 0.62, 0.672, 0.592, 0.572, 0.62, 0.692, 0.704, 0.628, 0.54, 0.568, 0.572, 0.524, 0.448, 0.576, 0.504, 0.452, 0.5, 0.444 ] }, { "key": "long", "name": "Długie dokumenty", "description": "Decyzje na podstawie dokumentów o długości 3–10 tys. tokenów.", "scores": [ 0.744, 0.736, 0.668, 0.668, 0.696, 0.668, 0.612, 0.6, 0.632, 0.56, 0.544, 0.572, 0.544, 0.588, 0.508, 0.504, 0.556, 0.076, 0.516, 0.488, 0.552, 0.464, 0.468, 0.468, 0.0, 0.348, 0.38, 0.384, 0.264 ] }, { "key": "realdoc", "name": "Rzeczywiste dokumenty", "description": "Pytania o treść autentycznych dokumentów urzędowych.", "scores": [ 0.992, 0.98, 0.952, 0.948, 0.976, 0.96, 0.94, 0.92, 0.94, 0.892, 0.888, 0.892, 0.908, 0.884, 0.848, 0.848, 0.832, 0.92, 0.848, 0.876, 0.824, 0.84, 0.828, 0.748, 0.728, 0.78, 0.788, 0.644, 0.348 ] }, { "key": "retrieval", "name": "Wyszukiwanie i RAG", "description": "Trafność fragmentu, wystarczalność kontekstu i oparcie twierdzenia w źródle.", "scores": [ 0.988, 0.976, 0.98, 0.976, 0.98, 0.96, 0.972, 0.94, 0.924, 0.956, 0.94, 0.86, 0.896, 0.916, 0.824, 0.92, 0.9, 0.86, 0.824, 0.796, 0.748, 0.716, 0.688, 0.712, 0.732, 0.52, 0.624, 0.568, 0.52 ] }, { "key": "routing", "name": "Kierowanie zgłoszeń", "description": "Wybór kolejki, intencji i priorytetu według podanych zasad.", "scores": [ 0.988, 0.976, 0.964, 0.976, 0.992, 0.928, 0.94, 0.968, 0.936, 0.932, 0.936, 0.912, 0.932, 0.892, 0.9, 0.9, 0.868, 0.916, 0.876, 0.86, 0.824, 0.82, 0.756, 0.756, 0.768, 0.784, 0.7, 0.688, 0.18 ] }, { "key": "rules", "name": "Reguły i terminy", "description": "Stosowanie reguł, terminów i warunków dotyczących kwot.", "scores": [ 0.442, 0.4, 0.382, 0.353, 0.34, 0.382, 0.348, 0.316, 0.305, 0.353, 0.34, 0.34, 0.284, 0.345, 0.28, 0.248, 0.324, 0.316, 0.236, 0.272, 0.296, 0.268, 0.252, 0.292, 0.252, 0.224, 0.208, 0.26, 0.252 ] }, { "key": "score", "name": "Ocena na skali", "description": "Wybór poziomu zgodnego z opisem skali.", "scores": [ 0.807, 0.772, 0.771, 0.799, 0.768, 0.747, 0.66, 0.688, 0.671, 0.683, 0.688, 0.672, 0.62, 0.594, 0.588, 0.608, 0.532, 0.64, 0.6, 0.552, 0.496, 0.572, 0.44, 0.492, 0.512, 0.444, 0.428, 0.412, 0.2 ] } ], "category_total": [ 0.821, 0.8088, 0.79, 0.785, 0.777, 0.763, 0.724, 0.7092, 0.701, 0.699, 0.6884, 0.6716, 0.664, 0.663, 0.6248, 0.6236, 0.619, 0.6156, 0.614, 0.5916, 0.5804, 0.5776, 0.5368, 0.527, 0.4924, 0.4872, 0.4816, 0.4592, 0.294 ], "family_models": [ "basal-1.0 · 4.5B", "basal-1.5 · 4.5B", "basal-1.5-max · 11B", "basal-1.0 · 1.5B", "basal-1.5-mini · 1.5B" ], "family_rows": [ { "name": "Zamknięty test polski", "description": "3 000 nowych decyzji; po zamrożeniu wag", "scores": [ 0.874, 0.931, 0.933, 0.889, 0.91 ] }, { "name": "Polskie decyzje", "description": "7 081 przypadków", "scores": [ 0.886, 0.923, 0.94, 0.851, 0.877 ] }, { "name": "Wiedza ogólna po polsku", "description": "3 079 przypadków", "scores": [ 0.737, 0.741, 0.794, 0.656, 0.667 ] } ], "samples": [ { "category": "abstain", "language": "pl", "state": "Fikcyjny rejestr magazynu. Produkt A: 18 sztuk na stanie, 7 zarezerwowanych. Produkt B: 9 sztuk na stanie. Rejestr nie podaje liczby rezerwacji produktu B. Towar dostępny do nowego zamówienia to stan pomniejszony o rezerwacje.", "question": "Ile sztuk produktu B można przeznaczyć na nowe zamówienie?", "options": [ "9 sztuk", "2 sztuki", "Nie da się ustalić na podstawie rejestru" ], "answer": "Nie da się ustalić na podstawie rejestru" }, { "category": "abstain", "language": "en", "state": "Synthetic handover log. Operator A covers the morning shift. Operator B has requested an exchange with Operator C for the afternoon shift. A request becomes effective only after approval by the shift lead. The log includes no approval and does not identify the original afternoon operator.", "question": "Who is assigned to the afternoon shift?", "options": [ "Operator B", "Operator C", "The log does not establish the assignment" ], "answer": "The log does not establish the assignment" }, { "category": "abstain", "language": "pl", "state": "Demonstracyjna specyfikacja urządzenia. Projekt dokumentu podaje limit 8 godzin pracy. Zatwierdzona wersja dokumentu podaje limit 6 godzin. Instrukcja procesu mówi, że wersja zatwierdzona zastępuje projekt.", "question": "Jaki limit czasu pracy obowiązuje według podanych dokumentów?", "options": [ "6 godzin", "8 godzin", "Nie da się ustalić" ], "answer": "6 godzin" }, { "category": "action", "language": "pl", "state": "Symulacja obsługi zgłoszenia. Procedura: najpierw sprawdź kompletność, następnie zweryfikuj uprawnienie, a na końcu zaproponuj rozwiązanie. Stan procesu: kompletność potwierdzona; uprawnienie jeszcze niesprawdzone. Dostępne narzędzia: sprawdź kompletność, sprawdź uprawnienie, przygotuj propozycję, zamknij zgłoszenie.", "question": "Jaki powinien być następny krok agenta?", "options": [ "Sprawdź uprawnienie", "Przygotuj propozycję", "Zamknij zgłoszenie" ], "answer": "Sprawdź uprawnienie" }, { "category": "action", "language": "en", "state": "Synthetic deployment exercise. Policy: a release may reach the staging environment only after both the build and automated checks pass. Production additionally requires an explicit approval. Current state: build passed, checks passed, staging not started, production approval absent.", "question": "Which action is allowed next?", "options": [ "Deploy to staging", "Deploy directly to production", "Skip the remaining steps" ], "answer": "Deploy to staging" }, { "category": "action", "language": "pl", "state": "Fikcyjne wywołanie usługi zwróciło kod 429 i nagłówek Retry-After: 20. Procedura pozwala na najwyżej trzy próby i wymaga odczekania czasu z nagłówka. Zakończyła się pierwsza próba; operacja jest bezpieczna do ponowienia.", "question": "Co powinien teraz zrobić agent?", "options": [ "Ponowić natychmiast", "Odczekać 20 sekund i ponowić", "Uznać operację za zakończoną sukcesem" ], "answer": "Odczekać 20 sekund i ponowić" }, { "category": "contract", "language": "pl", "state": "Umowa demonstracyjna: miesięczna opłata za usługę wynosi 200 punktów rozliczeniowych. Jeśli miesięczna dostępność spadnie poniżej 99%, przysługuje obniżka równa 10% tej opłaty. W badanym miesiącu dostępność wyniosła 98,5%. Innych obniżek nie przewidziano.", "question": "Jaką obniżkę przewidują podane warunki?", "options": [ "0 punktów", "20 punktów", "100 punktów" ], "answer": "20 punktów" }, { "category": "contract", "language": "en", "state": "Fictional service agreement. The supplier may use Subcontractor A for storage only. Access to message content requires separate written permission. No such permission has been granted. The proposed task is to let Subcontractor A read messages to prepare summaries.", "question": "Is the proposed task permitted by the stated agreement?", "options": [ "Yes, storage permission includes reading", "No, separate permission is required", "Yes, if the summaries are short" ], "answer": "No, separate permission is required" }, { "category": "contract", "language": "pl", "state": "Fikcyjna umowa wypożyczenia: podstawowy pakiet obejmuje 4 dni. Każdy rozpoczęty dodatkowy dzień kosztuje 15 punktów. Sprzęt pozostawał u odbiorcy przez 6 pełnych dni. Nie ma okresu bezpłatnego ani innych dopłat.", "question": "Jaka dopłata wynika z tej umowy?", "options": [ "15 punktów", "30 punktów", "90 punktów" ], "answer": "30 punktów" }, { "category": "judge", "language": "en", "state": "Synthetic reference: Plan A permits three devices and includes email support. It does not include telephone support. User question: how many devices are allowed and is telephone support included? Candidate answer: three devices are allowed and telephone support is included.", "question": "How should the candidate answer be assessed?", "options": [ "Fully correct", "Partly correct: the support claim is wrong", "Unsupported in every part" ], "answer": "Partly correct: the support claim is wrong" }, { "category": "judge", "language": "pl", "state": "Fikcyjna instrukcja: przy przeniesieniu projektu zachowaj jego nazwę, wskaż nowy katalog i wykonaj kopię zapasową. Pytanie dotyczy wszystkich wymaganych czynności. Odpowiedź A wymienia trzy czynności. Odpowiedź B wymienia zachowanie nazwy i wskazanie katalogu, lecz pomija kopię zapasową.", "question": "Która odpowiedź obejmuje wszystkie wymagane czynności?", "options": [ "Odpowiedź A", "Odpowiedź B", "Obie odpowiedzi" ], "answer": "Odpowiedź A" }, { "category": "judge", "language": "en", "state": "Synthetic reference: a completed form must contain a project code, a purpose and a requested amount. Candidate summary: the form needs a project code and a purpose. The summary makes no other claims.", "question": "What is the main issue with the summary?", "options": [ "It adds an unsupported requirement", "It omits the requested amount", "It contradicts the project code requirement" ], "answer": "It omits the requested amount" }, { "category": "long", "language": "pl", "state": "Skrócony, syntetyczny pakiet dokumentów.\n\nSekcja 1 — plan: test urządzenia A ma zakończyć się przed uruchomieniem urządzenia B. Zespół 1 odpowiada za test, Zespół 2 za uruchomienie.\n\nSekcja 2 — sprawa poboczna: urządzenie C przeszło test i otrzymało zgodę na pracę. Ta zgoda dotyczy wyłącznie C.\n\nSekcja 3 — raport: dla urządzenia A wykonano pomiary, lecz brakuje podpisu osoby zatwierdzającej. W tym procesie test uznaje się za zakończony dopiero po zatwierdzeniu.\n\nSekcja 4 — kolejność: Zespół 2 czeka na potwierdzenie zakończenia testu A. Nie wydano wyjątku od tej zasady.", "question": "Czy można już uruchomić urządzenie B?", "options": [ "Tak, ponieważ urządzenie C ma zgodę", "Tak, same pomiary wystarczą", "Nie, test A nie został zatwierdzony" ], "answer": "Nie, test A nie został zatwierdzony" }, { "category": "long", "language": "en", "state": "Short synthetic case file.\n\nSection 1 — request: Project A needs 12 units of Material X.\n\nSection 2 — warehouse: 20 units of Material X are stored, of which 6 are reserved for Project B. Reservations cannot be reassigned.\n\nSection 3 — unrelated shipment: 30 units of Material Y arrive tomorrow. They cannot substitute for Material X.\n\nSection 4 — rule: approve a request only if enough unreserved units of the requested material are available now.", "question": "Can the request for Project A be approved?", "options": [ "Yes, 14 unreserved units of Material X are available", "No, only 6 units are available", "Only after the Material Y shipment arrives" ], "answer": "Yes, 14 unreserved units of Material X are available" }, { "category": "long", "language": "pl", "state": "Skrócony, syntetyczny raport serwisowy.\n\nCzęść A — procedura: alert jest zakończony dopiero po naprawie i pomyślnym teście kontrolnym.\n\nCzęść B — archiwum: poprzedni alert dla modułu A został zakończony po teście. Ten wpis dotyczy wcześniejszego zdarzenia.\n\nCzęść C — bieżące zdarzenie: wymieniono uszkodzony element modułu A; test kontrolny nowej naprawy nie został jeszcze uruchomiony.\n\nCzęść D — notatka przekazania: zespół kolejnej zmiany ma wykonać brakujący test.", "question": "Jaki status wynika z procedury dla bieżącego alertu?", "options": [ "Zakończony", "Oczekuje na test kontrolny", "Nie wymaga żadnego działania" ], "answer": "Oczekuje na test kontrolny" }, { "category": "realdoc", "language": "en", "state": "Synthetic document written for this demonstration; it is not an official notice.\n\nNotice from Organisation A: applications for the fictional equipment programme are accepted electronically only. A submission consists of a completed form and an inventory list. A telephone call does not count as a submission.", "question": "Which submission method does this document specify?", "options": [ "Telephone only", "Electronic submission with the required attachments", "Any method without attachments" ], "answer": "Electronic submission with the required attachments" }, { "category": "realdoc", "language": "pl", "state": "Syntetyczny dokument demonstracyjny; nie jest to dokument urzędowy.\n\nKomunikat Organizacji A: magazyn demonstracyjny jest dostępny dla zespołów projektu od godziny 8:00 do 16:00. Pobranie sprzętu wymaga wpisania kodu projektu do rejestru. Pracownik dyżurny sprawdza zgodność kodu, ale nie zmienia godzin dostępu.", "question": "Jaką informację trzeba wpisać do rejestru przy pobraniu sprzętu?", "options": [ "Kod projektu", "Prywatny adres pracownika", "Numer dokumentu tożsamości" ], "answer": "Kod projektu" }, { "category": "realdoc", "language": "en", "state": "Synthetic document written for this demonstration; it is not a legal decision.\n\nDecision record of Organisation A: the request to expand the test area is accepted for two workstations. The separate request to expand the storage area remains pending because a floor plan is missing. The record makes no decision about opening hours.", "question": "Which request has been accepted?", "options": [ "Expansion of the storage area", "Expansion of the test area for two workstations", "A change to opening hours" ], "answer": "Expansion of the test area for two workstations" }, { "category": "retrieval", "language": "en", "state": "Synthetic knowledge snippets.\nPassage A: a project owner can export reports from the Reports menu.\nPassage B: account passwords are changed from Security settings.\nPassage C: billing history is visible to the billing role.\nUser question: where can a project owner export a report?", "question": "Which passage directly answers the user question?", "options": [ "Passage A", "Passage B", "Passage C" ], "answer": "Passage A" }, { "category": "retrieval", "language": "pl", "state": "Syntetyczny fragment bazy wiedzy: zadania ukończone można archiwizować. Archiwizacja zachowuje historię komentarzy.\nPytanie użytkownika: czy zarchiwizowane zadanie da się przywrócić? W zebranym materiale nie opisano przywracania.", "question": "Czy zebrany kontekst wystarcza do odpowiedzi na pytanie użytkownika?", "options": [ "Tak, można potwierdzić przywracanie", "Tak, można wykluczyć przywracanie", "Nie, potrzebny jest fragment o przywracaniu" ], "answer": "Nie, potrzebny jest fragment o przywracaniu" }, { "category": "retrieval", "language": "en", "state": "Synthetic source: notifications can be paused for one hour, four hours or until the next day. Pausing does not delete existing notifications.\nClaim to check: pausing notifications deletes all earlier notifications.", "question": "How does the claim relate to the source?", "options": [ "Supported", "Contradicted", "Not addressed" ], "answer": "Contradicted" }, { "category": "routing", "language": "en", "state": "Fictional support rules: login failures go to Account Access; questions about charges go to Billing; requests for new capabilities go to Product Feedback. Message from Customer A: I know my password, but the verification code never arrives and I cannot sign in.", "question": "Which queue should receive the message?", "options": [ "Account Access", "Billing", "Product Feedback" ], "answer": "Account Access" }, { "category": "routing", "language": "pl", "state": "Fikcyjna procedura: brak możliwości zapłaty kierujemy do Płatności, brak przesyłki do Dostawy, a pytania o parametry do Informacji o produkcie. Wiadomość Klienta A: parametry są jasne, chcę kupić produkt, lecz każda próba opłacenia koszyka kończy się błędem.", "question": "Do jakiej kolejki skierować główną prośbę?", "options": [ "Płatności", "Dostawy", "Informacje o produkcie" ], "answer": "Płatności" }, { "category": "routing", "language": "en", "state": "Fictional priority policy: service unavailable for all users = urgent; unavailable for one team without a workaround = high; cosmetic issue with a workaround = normal. Report: one team cannot use the export function and no workaround exists. Other teams can use it.", "question": "Which priority follows from this policy?", "options": [ "Urgent", "High", "Normal" ], "answer": "High" }, { "category": "rules", "language": "pl", "state": "Fikcyjny cennik: dostawa standardowa kosztuje 10 punktów. Jest bezpłatna, gdy wartość produktów po rabatach wynosi co najmniej 100 punktów. Klient A wybrał dostawę standardową. Produkty kosztują 120 punktów przed rabatem wynoszącym 25%.", "question": "Jaki koszt dostawy wynika z tych zasad?", "options": [ "0 punktów", "10 punktów", "25 punktów" ], "answer": "10 punktów" }, { "category": "rules", "language": "en", "state": "Fictional access rule: a temporary pass is valid for the three calendar days after the day it is issued. The issue day is excluded. A pass was issued on a Monday. There are no exceptions.", "question": "Is the pass valid on the immediately following Friday?", "options": [ "Yes", "No", "The issue hour is required to decide" ], "answer": "No" }, { "category": "rules", "language": "pl", "state": "Reguły demonstracyjne: zadanie zostaje automatycznie przydzielone tylko wtedy, gdy ma opiekuna i co najmniej dwie niezależne akceptacje. Zadanie A ma opiekuna, jedną akceptację i jeden komentarz bez akceptacji.", "question": "Czy zadanie A spełnia warunki automatycznego przydzielenia?", "options": [ "Tak", "Nie, brakuje drugiej akceptacji", "Nie, brakuje opiekuna" ], "answer": "Nie, brakuje drugiej akceptacji" }, { "category": "score", "language": "pl", "state": "Fikcyjna skala wpływu: 0 — brak utrudnienia; 1 — niewielkie utrudnienie z obejściem; 2 — zadanie zablokowane dla jednego zespołu; 3 — podstawowa usługa zablokowana dla wszystkich. Zespół A nie może ukończyć zadania i nie ma obejścia. Pozostałe zespoły pracują normalnie.", "question": "Który poziom skali pasuje do opisu?", "options": [ "0 — brak utrudnienia", "1 — niewielkie utrudnienie", "2 — blokada jednego zespołu", "3 — blokada wszystkich" ], "answer": "2 — blokada jednego zespołu" }, { "category": "score", "language": "en", "state": "Synthetic completeness rubric: 0 = none of the required fields; 1 = one field; 2 = two fields; 3 = all three fields. Required fields are purpose, project code and requested amount. The form contains a purpose and a project code, but no amount.", "question": "Which completeness level applies?", "options": [ "0", "1", "2", "3" ], "answer": "2" }, { "category": "score", "language": "en", "state": "Fictional relevance scale: low = unrelated topic; medium = same topic without answering the question; high = directly answers the question. Question: how can notifications be paused? Candidate passage: open Notification Settings and choose Pause, then select a duration.", "question": "What relevance level fits the passage?", "options": [ "Low", "Medium", "High" ], "answer": "High" } ], "sample_origin": { "kind": "synthetic-illustrations", "source": "Przykłady opracowane od podstaw na potrzeby strony basal.si5.pl.", "contains_source_documents": false, "used_for_ranking": false } }