{
  "date": "2026-10-22",
  "model": "basal-2.0-pico",
  "parameters_b": 0.8,
  "werdykt": {
    "n": 5000,
    "rows": [
      {
        "name": "basal-2.0-pico",
        "params": "0.8B",
        "macro": 0.6598,
        "pl": 0.6608,
        "en": 0.6588,
        "ci": [
          0.6484,
          0.6714
        ],
        "ours": true,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "PQ8_S; average of two final training checkpoints",
        "runtime": "basal 2.3.1 eager BF16"
      },
      {
        "name": "decider-2b (Mapika)",
        "params": "1.9B (Qwen3.5-2B merged checkpoint)",
        "macro": 0.5866,
        "pl": 0.5804,
        "en": 0.5928,
        "ci": [
          0.574,
          0.5994
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "533964dae8be954c5b5e19fa4948e48408094c1e (Mapika/decider-2b, main = v11)",
        "code_revision": "decider-ai 1.8.1 (PyPI)",
        "runtime": "decider.serve (uvicorn), bf16, CUDA graphs; torch 2.14.0+cu130, fla 0.5.2"
      },
      {
        "name": "krino-2b",
        "params": "2B (rank-16 LoRA + pointer head over Qwen/Qwen3.5-2B-Base)",
        "macro": 0.5662,
        "pl": 0.5524,
        "en": 0.58,
        "ci": [
          0.5534,
          0.5794
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "7463f8191e11581af4ab559faf353dece1c63759",
        "code_revision": "59b500e1b5ecaaf2c22cfc99c39af4bc48a7cb3c (Krino tag v0.1.0); kev fe64b1274ea7f80d4095866df90666abb03e9cf6",
        "runtime": "krino-serve (kev.serve + per-type temperatures choice 1.95 / noul 0.3 / score 0.5), bf16, CUDA graphs, fused FLA 0.5.2; kev uv lock torch 2.8.0+cu128, transformers 5.17.0"
      },
      {
        "name": "decision-2b (flymy-ai)",
        "params": "2B (openbmb/MiniCPM5-2B + adapter + pointer head + calibrator)",
        "macro": 0.5462,
        "pl": 0.4924,
        "en": 0.6,
        "ci": [
          0.5336,
          0.5578
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 431,
        "model_revision": "df57b75db927acc9ad91ec8115508c1e487086eb (flymy-ai/decision-2b-preview)",
        "code_revision": "n/a (code ships in the HF package)",
        "runtime": "package model.py load().decide() via pod/inproc_shim.py --kind flymy; torch 2.8.0 cu128, transformers 4.57.6, Python 3.11"
      },
      {
        "name": "qev-0.8b",
        "params": "0.8B",
        "macro": 0.544,
        "pl": 0.53,
        "en": 0.558,
        "ci": [
          0.5302,
          0.5564
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "labqoat/qev-0.8b @6334efd2",
        "code_revision": null,
        "runtime": "qev_server.py --backend fast --encoding per_question"
      },
      {
        "name": "jiwo-0.8b",
        "params": "0.75B (Qwen3.5-0.8B full fine-tune + linear readout)",
        "macro": 0.5416,
        "pl": 0.516,
        "en": 0.5672,
        "ci": [
          0.5286,
          0.554
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 112,
        "model_revision": "eljiwo/jiwo-0.8b @ 71b7911cf68f7df52fdc2dbf07de1e158bb807f2 (v2, main)",
        "code_revision": "jiwidi/jiwo @ e64d36227e02a36c0ea1bc3ec669003eb5a30c29",
        "runtime": "jiwo serve (author's FastAPI /v1/systemone server), jiwo[cuda] (torch 2.14.0, transformers 5.17.0, flash-linear-attention 0.5.2, kernels 0.16.1), JIWO_CUDA_GRAPHS=1"
      },
      {
        "name": "imajev-2b",
        "params": "1.9B (Qwen3.5-2B + rank-16 LoRA + 255-code decision readout)",
        "macro": 0.5342,
        "pl": 0.526,
        "en": 0.5424,
        "ci": [
          0.5216,
          0.5458
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 436,
        "model_revision": "mohit67890/imajev-2b @ e8cb9fc6e054a61ee3fdcff3cd20005ea6194588",
        "code_revision": "mohit67890/imajev @ 318e8dad55c12a1c6faddbec5f30489d2ee529ff",
        "runtime": "scripts/playground/server.py --backend torch (author's FastAPI /v1/systemone server, PyTorch + PEFT), pip extras [serve,torch], no flash-linear-attention (not in the extras)"
      },
      {
        "name": "jul-2b",
        "params": "2B (openbmb/MiniCPM5-2B + merged LoRA r16 + pointer head)",
        "macro": 0.5086,
        "pl": 0.4756,
        "en": 0.5416,
        "ci": [
          0.4946,
          0.5226
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "f556a6936089693ea68bfed3080c637e4853a3ff (usejul/jul-decision-minicpm5-2b, v2)",
        "code_revision": "bd41e5409c94f0197714635510d3e00079a59a34 (usejul/jul tag v0.5.1 = PyPI jul 0.5.1)",
        "runtime": "jul serve (stdlib HTTP, one forward pass at a time), torch backend"
      },
      {
        "name": "malkuth-2b",
        "params": "2B (LoRA + pointer head over empero-ai/Qwen3.8-2B-Distill)",
        "macro": 0.4998,
        "pl": 0.4816,
        "en": 0.518,
        "ci": [
          0.4874,
          0.5126
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 108,
        "model_revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087 (dhtocks/malkuth-2b)",
        "code_revision": "557598fc (jaredpalmer/kev)",
        "runtime": "kev.serve (kev uv lock, torch 2.8.0+cu128), bf16"
      },
      {
        "name": "Open-Jev-2B",
        "params": "1.9B (Qwen3.5-2B + rank-8 LoRA + scalar decision head)",
        "macro": 0.471,
        "pl": 0.448,
        "en": 0.494,
        "ci": [
          0.4584,
          0.4838
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 428,
        "model_revision": "ZefanCai/Open-Jev-2B @ 0c7aa498b1627be8da4acf34c863ff0ee0a92785",
        "code_revision": "Zefan-Cai/Open-Jev @ f46ff604f794b1932d1a6f34725c8163e550952a (the commit Benchmark Heaven ran)",
        "runtime": "python -m jev.server (author's own /v1/systemone server), torch 2.14.1+cu130, transformers 5.10.2, peft 0.19.1, no flash-linear-attention (not in the author's requirements)"
      },
      {
        "name": "EXAONE-4.0-1.2B-JEV",
        "params": "1.2B (LGAI-EXAONE/EXAONE-4.0-1.2B full fine-tune)",
        "macro": 0.4588,
        "pl": 0.3804,
        "en": 0.5372,
        "ci": [
          0.4462,
          0.4716
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "fde0914ef380c6fd2bfe3c48034bfa78f9fb094b (tag v0.3)",
        "code_revision": "147f7e8a3b0473b565e78f92e8290636c20197d6 (github.com/carrtesy/EXAONE-JEV)",
        "runtime": "serve/systemone_server.py on vLLM 0.30.0 cu129, torch 2.13.0 cu129, transformers 5.18.0"
      },
      {
        "name": "Ghafiri Qwen3.5-0.8B",
        "params": "0.8B (Qwen3.5-0.8B-Base full fine-tune)",
        "macro": 0.4428,
        "pl": 0.4432,
        "en": 0.4424,
        "ci": [
          0.4288,
          0.455
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "mghafiri/qwen3.5-0.8B-decision-model @ 4a9939034a27006b4b62ad5aed65a90b82d0d350",
        "code_revision": "(none: the bundled jevlite/ package in the HF repo)",
        "runtime": "our stdlib System One shim around jevlite.model.SystemOne(<snapshot>).system_one(state, questions) (model card usage), bf16 on CUDA, torch + transformers 5.17.0 (config's version), no flash-linear-attention / causal-conv1d"
      },
      {
        "name": "jevos-v5 (CPU)",
        "params": "17-layer MiniCPM5-family decoder (jevos), OpenVINO INT8 (~0.63 GB)",
        "macro": 0.4396,
        "pl": 0.398,
        "en": 0.4812,
        "ci": [
          0.4268,
          0.4526
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 193,
        "model_revision": "release asset jevos-v5-openvino-int8.zip (GitHub release jevos-v5; SHA256SUMS.txt checked)",
        "code_revision": "67c7846572c1a28562b4db77dee16ae3059d67f1 (tag jevos-v5)",
        "runtime": "jev serve (C++ binary from the release, OpenVINO bundled), CPU only, --threads 3"
      },
      {
        "name": "kev-0.6b",
        "params": "0.6B (LoRA r=16 + pointer head over Qwen/Qwen3-0.6B-Base)",
        "macro": 0.3752,
        "pl": 0.3644,
        "en": 0.386,
        "ci": [
          0.3618,
          0.3878
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 153,
        "model_revision": "dece6dba8d43f0f7ded45e9f5b9df12474d90843",
        "code_revision": "20fa6268c8ceb226530be2fb5266ab2c36b37724 (github.com/jaredpalmer/kev, the commit Benchmark Heaven served it from)",
        "runtime": "kev.serve @ 20fa626, KEV_DTYPE=bf16, kev uv lock torch 2.8.0+cu128, transformers 4.57.6, peft 0.21.0"
      },
      {
        "name": "vegaml-0.8b (Vega 0.8B)",
        "params": "0.8B frozen backbone (Qwen/Qwen3.5-0.8B) + 14.3M-parameter engine + 2 gated adapters",
        "macro": 0.3672,
        "pl": 0.3572,
        "en": 0.3772,
        "ci": [
          0.3548,
          0.3812
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "42ac8d4fa9efb61f96e53d80962cf28f6dfe667e",
        "code_revision": "none (vega_api.py / vega_common.py ship in the HF snapshot)",
        "runtime": "shim.py around vega_api.Vega(<snapshot>, device=cuda), adapters auto, views 1, alpha 0.1; backbone bf16 (the code's choice on sm>=80), sdpa; torch 2.14.1+cu130, transformers 5.20.2, flash-linear-attention 0.5.2"
      },
      {
        "name": "decision-fast (0.6B)",
        "params": "0.6B (Qwen/Qwen3-0.6B-Base + LoRA adapter + pointer head, 606.7M total, 10.6M trained)",
        "macro": 0.3546,
        "pl": 0.3488,
        "en": 0.3604,
        "ci": [
          0.341,
          0.367
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 451,
        "model_revision": "4225d41c66119fe28e95a2631bb0103decae6d56 (flymy-ai/decision-fast-preview, checkpoint qwen3_06b_headfirst_ep2a_v53)",
        "code_revision": "n/a (code ships in the HF package)",
        "runtime": "package model.py load().decide() via pod/inproc_shim.py --kind flymy; torch 2.8.0 cu128, transformers 4.57.6, peft 0.15.2, Python 3.11"
      },
      {
        "name": "Tacet Sonata",
        "params": "144M (jhu-clsp/mmBERT-small encoder + 2-layer head)",
        "macro": 0.35,
        "pl": 0.3472,
        "en": 0.3528,
        "ci": [
          0.3364,
          0.3624
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "26f447c35881a80020fe557c95ceb8c61bc3fd12 (codepawl/tacet-sonata)",
        "code_revision": "0f0aefb9e6a66ed4436efc0beaee4e1839547436 (github.com/codepawl/tacet = PyPI tacet 0.3.0)",
        "runtime": "tacet serve (FastAPI/uvicorn, micro-batches of up to 8), torch CUDA bf16 autocast, max length 4096"
      },
      {
        "name": "watt-flash-0.1",
        "params": "140M (mmBERT-small encoder + option scorer)",
        "macro": 0.3476,
        "pl": 0.3312,
        "en": 0.364,
        "ci": [
          0.335,
          0.3596
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 105,
        "model_revision": "d49a3a880b0201b325f2d2299d4c36ffc8735477",
        "code_revision": "e9863f06230ad874a5a2cfca3af922e45161a4d7",
        "runtime": "python -m watt_flash.serve (repo's own /v1/systemone server, HTTPServer single-threaded), uv lock: torch 2.14.0, transformers 5.17.0, bf16 autocast"
      },
      {
        "name": "Quyet-1.0-Small",
        "params": "328M (aisingapore/SEA-LION-ModernBERT-300M encoder + 2-layer refinement head)",
        "macro": 0.3348,
        "pl": 0.306,
        "en": 0.3636,
        "ci": [
          0.3212,
          0.3466
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "f007dfeb4874f0f3b3a3cc0a63f4d4192164c1c2 (chinhnc/Quyet-1.0-Small)",
        "code_revision": "9811cb35aa37772f11fa1314d7f6d70de05a3261 (github.com/ncchinh/quyet tag v1.0.2 = PyPI quyet 1.0.2)",
        "runtime": "quyet 1.0.2 in process via shim.py --kind quyet (stdlib HTTP, serial); CUDA bf16 autocast"
      },
      {
        "name": "liquid-d1-omni-600m",
        "params": "587M (381M trunk + decision head on LFM2.5-Encoder-350M, 94M vision, 112M audio; text path only used)",
        "macro": 0.3334,
        "pl": 0.3044,
        "en": 0.3624,
        "ci": [
          0.3208,
          0.346
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 34,
        "model_revision": "02b55d7076f15129e59ab3f94783f32c4b088674",
        "code_revision": "none (remote code modeling_d1.py etc. ships in the HF snapshot, read before use)",
        "runtime": "shim.py around AutoModel.from_pretrained(trust_remote_code=True, dtype=float16).system_one on CUDA; torch 2.14.1+cu130, transformers 5.20.2"
      },
      {
        "name": "lev-350m",
        "params": "361M (LiquidAI/LFM2.5-350M + LoRA + decision head)",
        "macro": 0.3316,
        "pl": 0.3132,
        "en": 0.35,
        "ci": [
          0.3178,
          0.3436
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 140,
        "model_revision": "franckverrot/lev-350m @ ab08ad8b8f346994d983152917e114224f6adac7",
        "code_revision": "franckverrot/lev @ c48a945dbf629998d7458dcc5c16f58df964db94",
        "runtime": "python -m lev.serve (author's FastAPI /v1/systemone server, uvicorn), uv sync --frozen --extra serve (torch 2.8.0, transformers 5.17.0), fp32 as the author's default"
      },
      {
        "name": "phocinae-largha-150m-v1",
        "params": "144.3M (jhu-clsp/mmBERT-small + decision head)",
        "macro": 0.3226,
        "pl": 0.3136,
        "en": 0.3316,
        "ci": [
          0.3098,
          0.3358
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "a57030322df13c8c55ab7a99b8b7b176e2c5aa85 (Phocinae/Phocinae-Largha-150M-v1)",
        "code_revision": "PyPI phocinae-server 0.1.8 (github.com/Phocinae/phocinae-server)",
        "runtime": "phocinae-server 0.1.8 Engine in process via shim.py --kind phocinae; pure-torch forward, CUDA fp16 autocast, torch.compile reduce-overhead"
      },
      {
        "name": "WaterSheep",
        "params": "0.15B (ModernBERT-base encoder + calibrated heads)",
        "macro": 0.3172,
        "pl": 0.282,
        "en": 0.3524,
        "ci": [
          0.3048,
          0.3284
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "samratduttaofficial/WaterSheep @ 236de6496b530d37e36a8ac63ff8412e92940f6e (export watersheep-20260928-125452, v0.1.0)",
        "code_revision": "SamratDuttaOfficial/WaterSheep @ aa308f7778a06bad77c1adb3e3109e27bed03736",
        "runtime": "the author's watersheep package (WaterSheep.load(folder, cuda).ask(request), as `watersheep --serve`), bf16 autocast; torch + transformers>=5 from its pyproject"
      },
      {
        "name": "openjev-verdict-1.4",
        "params": "151M (GLiClass ModernBERT-base)",
        "macro": 0.306,
        "pl": 0.294,
        "en": 0.318,
        "ci": [
          0.2922,
          0.3188
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "8af2496eb63c7fa66d7d234e1f62629380030eb4",
        "code_revision": "30f15564821626ca5c1ad5b2638c4eb7078787dd (Heman10x-NGU/Verdict-open-jev, the v1.4 engine)",
        "runtime": "shim.py around core.engine_encoder.DecisionEngine(device=cuda): GLiClassModel FP32 on the GPU (torch path), calibrator.json per option count; torch 2.14.1, transformers 5.20.2, gliclass 0.1.21 (--device cpu = the ONNX path BH ran)"
      },
      {
        "name": "verdict-small",
        "params": "118M (intfloat/multilingual-e5-small bi-encoder, fine-tuned)",
        "macro": 0.3006,
        "pl": 0.304,
        "en": 0.2972,
        "ci": [
          0.2876,
          0.3134
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 0,
        "model_revision": "Manav2op/verdict-small @ 085a41e0ea269f00483dd202a286cfb68f7aa63a (the registry's verdict-small)",
        "code_revision": "Manavarya09/verdict @ 76bc1fe9ba87ec5b746ed33faac48812f1d1a4ca (verdictml 0.1.0)",
        "runtime": "verdict serve (author's FastAPI /v1/systemone server), verdictml[serve] from the pinned commit, sentence-transformers 6.2.0, torch on CUDA"
      },
      {
        "name": "kev-0.5b",
        "params": "0.5B (LoRA + pointer head over Qwen/Qwen2.5-0.5B)",
        "macro": 0.3002,
        "pl": 0.2896,
        "en": 0.3108,
        "ci": [
          0.2868,
          0.3124
        ],
        "ours": false,
        "n": 5000,
        "failed_calls": 153,
        "model_revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd",
        "code_revision": "20fa6268c8ceb226530be2fb5266ab2c36b37724 (github.com/jaredpalmer/kev, the commit Benchmark Heaven served it from)",
        "runtime": "kev.serve @ 20fa626, KEV_DTYPE=bf16, kev uv lock torch 2.8.0+cu128, transformers 4.57.6, peft 0.21.0"
      }
    ],
    "category": {
      "abstain": 0.544,
      "action": 0.592,
      "contract": 0.542,
      "judge": 0.752,
      "long": 0.582,
      "realdoc": 0.848,
      "retrieval": 0.886,
      "routing": 0.912,
      "rules": 0.316,
      "score": 0.624
    },
    "metric": "macro accuracy across 20 category/language cells"
  },
  "public": {
    "tests": [
      "Benchmark Heaven (public)",
      "JevExtra",
      "JevSpatial",
      "Rules-7",
      "JevBench · accuracy",
      "JevBench · coherence"
    ],
    "rows": [
      {
        "model": "basal-pico 0.8B",
        "values": [
          75.3,
          68.4,
          68.7,
          68.6,
          83.9,
          74.3
        ]
      },
      {
        "model": "qev-0.8b",
        "values": [
          77.9,
          71.8,
          74.2,
          63.8,
          85.1,
          70.7
        ]
      },
      {
        "model": "krino-2b",
        "values": [
          69.7,
          64.7,
          75.0,
          62.2,
          84.3,
          76.01
        ]
      },
      {
        "model": "jiwo-0.8b",
        "values": [
          74.0,
          66.4,
          69.3,
          57.8,
          85.5,
          70.91
        ]
      }
    ]
  },
  "scope": "25 measured alternatives, <=2B class; excludes internal nano E2F (~5B total) and the replaced 0.6B pico preview. Not an exhaustive global leaderboard. Model/server limits and failures are retained in the comparison.",
  "caveats": [
    "Werdykt is authored by the basal project. These are our runs, not author submissions.",
    "Release weights were selected after both candidate Werdykt scores were known; intervals overlap.",
    "Public scores are local evaluations from the release card and rivals report, not a claim of placement on an official board."
  ],
  "source_release": "https://github.com/rkinas/basal/releases/tag/v2.3.2",
  "model_card": "https://huggingface.co/Remek/basal-2.0-pico",
  "model_card_status_at_article_build": "not yet anonymously accessible",
  "checkpoint_selection": {
    "candidate_scores": [
      0.6598,
      0.6492
    ],
    "selected_after_scores_known": true
  },
  "runtime_measurements": {
    "gpu": "H100 80 GB",
    "mode": "eager BF16",
    "engine_version": "2.3.1",
    "p50_ms": 47,
    "concurrency_16_decisions_per_second": 32.4,
    "cpu": {
      "processor": "Intel Xeon Platinum 8460Y+",
      "threads": 16,
      "engine_version": "2.3.2",
      "format": "GGUF Q8_0",
      "example_tokens": 232,
      "example_questions": 2,
      "seconds_range": [
        0.46,
        0.51
      ]
    },
    "gguf_file_mb": 812,
    "gguf_agreement_vs_bf16": {
      "n": 1700,
      "cpu": 0.9929,
      "h100": 0.9924
    }
  },
  "calibration": {
    "werdykt_ece_before_final_calibration": 0.143,
    "final_calibration_separate_frozen_test": {
      "n": 8560,
      "target_1_percent": {
        "coverage": 0.35,
        "observed_error": 0.0093
      },
      "target_5_percent": {
        "coverage": 0.582,
        "observed_error": 0.0438
      }
    }
  }
}