Sieve
Collection
Decision models that return calibrated probabilities for typed questions about text or JSON, without generating text. • 24 items • Updated • 10
Dataset Viewer
The dataset viewer is not available for this split.
Cannot load the dataset split (in streaming mode) to extract the first rows.
Error code: StreamingRowsError
Exception: CastError
Message: Couldn't cast
engine: string
engine_options: struct<base_url: string, model: string>
child 0, base_url: string
child 1, model: string
model_source: struct<kind: string, base_url: string, model: string, request_options: struct<>, policy: string>
child 0, kind: string
child 1, base_url: string
child 2, model: string
child 3, request_options: struct<>
child 4, policy: string
loaded_seconds: double
frozen_corpus_sha256: null
rows_path: list<item: string>
child 0, item: string
runner_sha256: string
latency: string
successful_request_latency_ms: struct<median: double, p95: double, mean: double>
child 0, median: double
child 1, p95: double
child 2, mean: double
edition: string
counts: struct<ok: int64, unsupported: int64>
child 0, ok: int64
child 1, unsupported: int64
note: string
benchmarks: list<item: struct<catalog_id: int64, dataset: string, requests: int64, answered: int64, unsupported: (... 6276 chars omitted)
child 0, item: struct<catalog_id: int64, dataset: string, requests: int64, answered: int64, unsupported: int64, err (... 6264 chars omitted)
child 0, catalog_id: int64
child 1, dataset: string
child 2, requests: int64
child 3, answered: int64
child 4, unsupported: int64
child 5, errors: int64
child 6, abstained: int64
child 7, pending: int64
child 8, scored_requests: int64
child 9, metric: string
child 10, score: double
child 11, reference_same_cases: null
child 12, median_ms: dou
...
iSarcasmEval-A-Ar: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 3, iSarcasmEval-A-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 4, iSarcasmEval-B-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 5, iSarcasmEval-C-Ar: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 6, iSarcasmEval-C-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 7, GSM8K-10choice: struct<metric: string, score: double, scored_requests: int64>
child 0, metric: string
child 1, score: double
child 2, scored_requests: int64
child 8, GSM8K-4choice: struct<metric: string, score: double, scored_requests: int64>
child 0, metric: string
child 1, score: double
child 2, scored_requests: int64
to
{'engine': Value('string'), 'counts': {'ok': Value('int64'), 'unsupported': Value('int64')}, 'successful_request_latency_ms': {'median': Value('float64'), 'p95': Value('float64'), 'mean': Value('float64')}, 'benchmarks': List({'catalog_id': Value('int64'), 'dataset': Value('string'), 'requests': Value('int64'), 'answered': Value('int64'), 'unsupported': Value('int64'), 'errors': Value('int64'), 'abstained': Value('int64'), 'pending': Value('int64'), 'scored_requests': Value('int64'), 'metric': Value('string'), 'score': Value('float64'), 'reference_same_cases': Value('null'), 'median_ms': Value('float64'), 'detail': {'field_accuracy': Value('float64'), 'case_exact_accuracy': Value('float64'), 'custom_metrics': {'ndcg_at_10': Value('float64'), 'mrr': Value('float64'), 'recall_at_10': Value('float64'), 'candidate_recall': Value('float64'), 'scorable_candidate_recall': Value('float64'), 'candidates_scored': Value('float64'), 'candidates_retrieved': Value('int64'), 'bm25_ndcg_at_10': Value('float64'), 'quality_quality': Value('float64'), 'quality_cost_usd': Value('float64'), 'quality_utility': Value('float64'), 'quality_oracle_optimal': Value('float64'), 'quality_utility_regret': Value('float64'), 'cost_aware_quality': Value('float64'), 'cost_aware_cost_usd': Value('float64'), 'cost_aware_utility': Value('float64'), 'cost_aware_oracle_optimal': Value('float64'), 'cost_aware_utility_regret': Value('float64'), 'value_regret': Value('float64'), 'normalized_rank_regret': Value('float6
...
lue('int64'), 'mean': Value('float64')}}, 'cluster_macro_accuracy': Value('float64'), 'positive_f1_by_field': {'sarcastic': Value('float64'), 'sarcasm': Value('float64'), 'irony': Value('float64'), 'satire': Value('float64'), 'understatement': Value('float64'), 'overstatement': Value('float64'), 'rhetorical_question': Value('float64')}, 'category_macro_f1': Value('float64'), 'scored_fields': Value('int64'), 'chance_on_rows': Value('float64')}, 'tracks': {'RouterBench-0shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'RouterBench-5shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-B-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'GSM8K-10choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'GSM8K-4choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}}}), 'note': Value('string'), 'edition': Value('string')}
because column names don't match
Traceback: Traceback (most recent call last):
File "/src/services/worker/src/worker/utils.py", line 147, in get_rows_or_raise
return get_rows(
dataset=dataset,
...<4 lines>...
column_names=column_names,
)
File "/src/libs/libcommon/src/libcommon/utils.py", line 272, in decorator
return func(*args, **kwargs)
File "/src/services/worker/src/worker/utils.py", line 127, in get_rows
rows_plus_one = list(itertools.islice(safe_iter(ds, dataset=dataset), rows_max_number + 1))
File "/src/services/worker/src/worker/utils.py", line 483, in safe_iter
yield from ds.decode(False) if ds.features else ds
File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 2951, in __iter__
for key, example in ex_iterable:
^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 2461, in __iter__
for key, pa_table in self._iter_arrow():
~~~~~~~~~~~~~~~~^^
File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 2486, in _iter_arrow
for key, pa_table in self.ex_iterable._iter_arrow():
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 547, in _iter_arrow
for key, pa_table in iterator:
^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 430, in _iter_arrow
for key, pa_table in self.generate_tables_fn(**gen_kwags):
~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/datasets/packaged_modules/json/json.py", line 343, in _generate_tables
self._cast_table(pa_table, json_field_paths=json_field_paths),
~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/datasets/packaged_modules/json/json.py", line 132, in _cast_table
pa_table = table_cast(pa_table, self.info.features.arrow_schema)
File "/usr/local/lib/python3.14/site-packages/datasets/table.py", line 2378, in table_cast
return cast_table_to_schema(table, schema)
File "/usr/local/lib/python3.14/site-packages/datasets/table.py", line 2306, in cast_table_to_schema
raise CastError(
...<3 lines>...
)
datasets.table.CastError: Couldn't cast
engine: string
engine_options: struct<base_url: string, model: string>
child 0, base_url: string
child 1, model: string
model_source: struct<kind: string, base_url: string, model: string, request_options: struct<>, policy: string>
child 0, kind: string
child 1, base_url: string
child 2, model: string
child 3, request_options: struct<>
child 4, policy: string
loaded_seconds: double
frozen_corpus_sha256: null
rows_path: list<item: string>
child 0, item: string
runner_sha256: string
latency: string
successful_request_latency_ms: struct<median: double, p95: double, mean: double>
child 0, median: double
child 1, p95: double
child 2, mean: double
edition: string
counts: struct<ok: int64, unsupported: int64>
child 0, ok: int64
child 1, unsupported: int64
note: string
benchmarks: list<item: struct<catalog_id: int64, dataset: string, requests: int64, answered: int64, unsupported: (... 6276 chars omitted)
child 0, item: struct<catalog_id: int64, dataset: string, requests: int64, answered: int64, unsupported: int64, err (... 6264 chars omitted)
child 0, catalog_id: int64
child 1, dataset: string
child 2, requests: int64
child 3, answered: int64
child 4, unsupported: int64
child 5, errors: int64
child 6, abstained: int64
child 7, pending: int64
child 8, scored_requests: int64
child 9, metric: string
child 10, score: double
child 11, reference_same_cases: null
child 12, median_ms: dou
...
iSarcasmEval-A-Ar: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 3, iSarcasmEval-A-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 4, iSarcasmEval-B-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 5, iSarcasmEval-C-Ar: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 6, iSarcasmEval-C-En: struct<metric: string, score: null, scored_requests: int64>
child 0, metric: string
child 1, score: null
child 2, scored_requests: int64
child 7, GSM8K-10choice: struct<metric: string, score: double, scored_requests: int64>
child 0, metric: string
child 1, score: double
child 2, scored_requests: int64
child 8, GSM8K-4choice: struct<metric: string, score: double, scored_requests: int64>
child 0, metric: string
child 1, score: double
child 2, scored_requests: int64
to
{'engine': Value('string'), 'counts': {'ok': Value('int64'), 'unsupported': Value('int64')}, 'successful_request_latency_ms': {'median': Value('float64'), 'p95': Value('float64'), 'mean': Value('float64')}, 'benchmarks': List({'catalog_id': Value('int64'), 'dataset': Value('string'), 'requests': Value('int64'), 'answered': Value('int64'), 'unsupported': Value('int64'), 'errors': Value('int64'), 'abstained': Value('int64'), 'pending': Value('int64'), 'scored_requests': Value('int64'), 'metric': Value('string'), 'score': Value('float64'), 'reference_same_cases': Value('null'), 'median_ms': Value('float64'), 'detail': {'field_accuracy': Value('float64'), 'case_exact_accuracy': Value('float64'), 'custom_metrics': {'ndcg_at_10': Value('float64'), 'mrr': Value('float64'), 'recall_at_10': Value('float64'), 'candidate_recall': Value('float64'), 'scorable_candidate_recall': Value('float64'), 'candidates_scored': Value('float64'), 'candidates_retrieved': Value('int64'), 'bm25_ndcg_at_10': Value('float64'), 'quality_quality': Value('float64'), 'quality_cost_usd': Value('float64'), 'quality_utility': Value('float64'), 'quality_oracle_optimal': Value('float64'), 'quality_utility_regret': Value('float64'), 'cost_aware_quality': Value('float64'), 'cost_aware_cost_usd': Value('float64'), 'cost_aware_utility': Value('float64'), 'cost_aware_oracle_optimal': Value('float64'), 'cost_aware_utility_regret': Value('float64'), 'value_regret': Value('float64'), 'normalized_rank_regret': Value('float6
...
lue('int64'), 'mean': Value('float64')}}, 'cluster_macro_accuracy': Value('float64'), 'positive_f1_by_field': {'sarcastic': Value('float64'), 'sarcasm': Value('float64'), 'irony': Value('float64'), 'satire': Value('float64'), 'understatement': Value('float64'), 'overstatement': Value('float64'), 'rhetorical_question': Value('float64')}, 'category_macro_f1': Value('float64'), 'scored_fields': Value('int64'), 'chance_on_rows': Value('float64')}, 'tracks': {'RouterBench-0shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'RouterBench-5shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-B-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'GSM8K-10choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'GSM8K-4choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}}}), 'note': Value('string'), 'edition': Value('string')}
because column names don't matchNeed help to make the dataset viewer work? Make sure to review how to configure the dataset viewer, and open a discussion for direct support.
Sieve-4B: Decision Index results
The complete run of Sieve-4B on the public suite of the Decision Index 0.3, made with the official kit
apolinario/decision-index at 9eb2dbe (0.3); the kit's own scorer produced the scores.
| Leaderboard | Full score 40.96, public score 50.06 (edition 0.4.1, board data of 2026-10-26) |
| This run | Decision Index 0.3: public index 50.06 (raw index 62.10, breadth 48.32) |
| Requests | all 140,178 scoreable requests answered: 0 unsupported, 0 errors |
The Full score ranks the board: it comes from the maintainers' private tests, 60% tests of the same skills as the public benchmarks and 40% decision tasks from new domains.
| area | skill (×100) |
|---|---|
| Knowledge & Reasoning | 33.9 |
| Language Understanding | 53.9 |
| Retrieval & Classification | 61.6 |
| Tools & Automation | 66.8 |
| Arts & Human Taste | 28.3 |
Files (runs/Sieve-4B/)
| file | contents |
|---|---|
scores.json |
the kit's scores: "complete": true, edition 0.3, index, areas, per-benchmark values |
index.json, benchmark-summary.json |
per-benchmark raw, skill and coverage; native metrics and counts |
environment.json, status.json |
the runner's environment and final status for the 0.3 step |
shards/0.2.1/, shards/0.3/ |
the runner's environment and final status for each of the six shards of both steps |
results.jsonl.gz |
one result per request: run id, status, the model's answers with a probability for every option, and timings |
RUN_NOTES.md |
how the run was made, step by step, with checksums |
results.jsonl.gz is in the runner's --compact form: the payload field (benchmark text) and the raw_output field
are left out, because several suite sources do not allow their text to be republished.
Scoring this file with python -m decision_index score --edition 0.3 --results runs/Sieve-4B/results.jsonl.gz
reproduces scores.json, index.json and benchmark-summary.json, apart from their generation time.
How the run was made
- Two steps (2026-10-08 to 09), on two NVIDIA A100 80GB PCIe GPUs: the complete 0.2.1 suite (150,759 rows), then
the rows 0.3 adds (the 2,638 rebuilt GSM8K requests). Each step was split into six shards by run id, one per server
process, three processes per GPU. See
RUN_NOTES.md. - Model.
sthanika-ai/Sieve-4B, served by its own/v1/systemoneserver (thensieve4b.serverin the model repo, nowsieve.codes.serverin github.com/sthanika-ai/Sieve) through the kit'shttpengine. The adapter is merged into the bf16 backbone, the answer head runs in fp32, and the temperatures are the ones in itssieve_config.json. - Nothing is truncated, no option is dropped, and no prompt is changed per benchmark.
- Downloads last month
- 319
