{
  "schema_version": "1.0",
  "snapshot_utc": "2026-10-27T18:06:24.511077+00:00",
  "catalogue_tasks": 31,
  "evaluated_tasks": 24,
  "deferred_tasks": 7,
  "domains": 10,
  "models": [
    "laya",
    "intern_decision_4b",
    "llama8b",
    "minitaur8b",
    "qwen35_4b"
  ],
  "full_sample_protocol_cells": 120,
  "jev_status": "api_full_sample_modes_27_of_27",
  "scope": "Fixed first-paper 24-task five-open-model evaluation; 31-task catalogue retained. Full protocol coverage is not all-mode or all-metric completeness.",
  "unified_cli": {
    "script": "scripts/benchmark_paper_tasks.py",
    "operations": {
      "list": [
        "horizon-predictive",
        "mouselab-predictive",
        "mouselab-generative",
        "twostep-generative",
        "horizon-generative",
        "probabilistic-posterior"
      ],
      "export": [
        "horizon-predictive",
        "mouselab-predictive",
        "probabilistic-posterior"
      ],
      "score": [
        "horizon-predictive",
        "mouselab-predictive",
        "probabilistic-posterior"
      ],
      "predict": [
        "horizon-predictive",
        "mouselab-predictive"
      ],
      "run": [
        "mouselab-generative",
        "twostep-generative",
        "horizon-generative",
        "probabilistic-posterior"
      ]
    },
    "adapter_contracts": {
      "event": "create_agent(device) -> object.posterior_probability(state, question), scalar event probability in [0,1]",
      "action": "create_agent(device) -> object.choice_probabilities(state, question, output_keys), normalized probabilities in supplied legal-key order"
    },
    "all_task_cpu_validation_driver": "scripts/validate_cpu_reuse_24.py",
    "note": "The 24-task/27-mode CPU validation driver dispatches to task-specific runners. It is not a universal five-neural-model inference CLI."
  },
  "tasks": [
    {
      "task_number": 1,
      "task_id": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "domain": "Evidence & probability",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "prior base rate; urn likelihood and observed color",
          "original_papers": [
            {
              "citation": "CogBench ICML 2024",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "prior; urn composition and observed color",
          "output": "Posterior probability of the target urn",
          "history": "This is an event probability; when Bayes errors are all zero, the original strict trimmed score is undefined; generated items are not human data",
          "split_protocol": "Original code has 200 runs×2 conditions=400 items; the first 100 runs are reported separately at paper scale; no train/test",
          "metrics": "Original trimmed 1−MAE; prior/likelihood regression weights; full-data MAE and event Brier/NLL reported separately",
          "baselines": [
            {
              "name": "Bayes",
              "status": "author_reference",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Analytic normative posterior; the added full-set metric and the original trimmed score are reported separately"
            }
          ],
          "adaptation_note": "This is an event probability; when Bayes errors are all zero, the original strict trimmed score is undefined; generated items are not human data",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench",
              "availability": "public",
              "note": "Original source code is available locally; official version/scoring retained"
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "prior wheel proportions and urn red/blue ball proportions varied separately",
          "original_papers": [
            {
              "citation": "Dasgupta et al. (2020). A theory of learning to infer. Psychological Review 127:412.",
              "url": "https://doi.org/10.1037/rev0000178",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Full prior; likelihood and observed ball color",
          "output": "event posterior probability 0–1",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "The original query defaults to 200 runs×two conditions; the paper describes 100 simulations; existing local reports list them separately and do not conflate them under the same name.",
          "metrics": "original prior/likelihood weight regression; original trimmed error; full MAE/Brier is an analysis this project reports separately",
          "baselines": [
            {
              "name": "Bayes posterior and uniform reference",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "explicit posterior is not directly replaced by a two-action softmax; the original strict trimming may be undefined for a zero-error reference.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/ProbabilisticReasoning",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/ProbabilisticReasoning/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/benchmark_paper_tasks.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--predictions"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "TASKS",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*GENERATIVE_TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/benchmark_paper_tasks.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "probabilistic-posterior",
      "supported_unified_cli": [
        {
          "key": "probabilistic-posterior",
          "operations": [
            "list",
            "export",
            "score",
            "run"
          ],
          "specification": {
            "paper": "https://proceedings.mlr.press/v235/coda-forno24a.html",
            "task_family": "evidence_integration_and_planning",
            "data_kind": "new_program_generated_urn_observations_not_human_data",
            "version": "1",
            "runs_code_default": 200,
            "queries_code_default": 400,
            "runs_paper_statement": 100,
            "queries_paper_scale": 200,
            "scale_protocol": "Keep all 200 code-default runs; predeclare first 100 as paper-scale report.",
            "output": "reported_event_probability = P(named target urn | visible prior and observed color)",
            "metric": "current_official_strict_q95_posterior_error_and_prior_likelihood_weights",
            "additional_metrics": "Full posterior MAE, generated-event Brier/NLL, separately rounded report.",
            "source": "external/CogBench/Experiments/ProbabilisticReasoning/query.py",
            "score_status": "original_generator_and_current_scorer_with_explicit_event_probability_adapter",
            "protocol_deviations": "New replay seeds and typed event interface; publication/code scale and regression-intercept differences are disclosed, not silently reconciled.",
            "task_state": "full_static_export_scoring_and_external_callback_available",
            "adapter_contract": "create_agent(device) returns an object with posterior_probability(state, question); return a scalar probability in [0,1]."
          }
        }
      ],
      "result_modes": [
        "generated_explicit_posterior"
      ],
      "result_protocols": [
        "cogbench_generated_posterior_200x2",
        "documented_task_adapter"
      ],
      "available_metric_definitions": [
        [
          "event_answer_brier",
          "event_brier",
          "The observed ball was generated from the left urn; saved hidden event labels are used only for scoring.",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "event_brier_additional",
          "true_generated_source_urn",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "likelihood_logodds_weight",
          "likelihood_weight_original_no_intercept_OLS",
          "original_OLS_coefficient",
          "coefficient",
          "response_weighted_mean"
        ],
        [
          "mae",
          "posterior_mae",
          "analytic_Bayes_posterior",
          "probability_0_to_1",
          "response_weighted_mean"
        ],
        [
          "mae",
          "posterior_mae_full_unclipped_additional",
          "analytic_Bayes_posterior",
          "probability_0_to_1",
          "response_weighted_mean"
        ],
        [
          "prior_logodds_weight",
          "prior_weight_original_no_intercept_OLS",
          "original_OLS_coefficient",
          "coefficient",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 59,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "bayesian_urn_inference",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 400,
            "expected_full_input_count": 400,
            "original_scored_count": 400,
            "expected_scored_count": 400,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "claude_opus_5_5",
        "claude_sonnet_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 2,
      "task_id": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "domain": "Evidence & probability",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "two cues with five levels each; nine-level outcome; additive and non-additive tasks; verbal and numerical representations; phase after feedback stops",
          "original_papers": [
            {
              "citation": "Collsiöö, A., Juslin, P. & Winman, A. Is numerical information always beneficial? verbal and numerical cue-integration in additive and non-additive tasks. Cognition 240 , 105584 (2023).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib97",
              "relation": "source_study"
            }
          ],
          "input": "Current cues and earlier training feedback",
          "output": "nine-level outcome classification and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; outcome prediction error; generalization after feedback stops; cue weights. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Linear regression model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx7",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Original outcome levels are ordinal; level numbers are not treated as actual continuous concentrations",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_compact.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_compact.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "multiple_cue_judgment",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "compact_full_history",
        "original_full_prefix_typed",
        "psych101-extended-full-prefix-typed-v1"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "objective_label_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "mae",
          "native_rating_MAE",
          "original_human_numeric_response",
          "original_concentration_rating_steps_1_to_9",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 335,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "multiple_cue_judgment",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 6136,
            "expected_full_input_count": 6136,
            "original_scored_count": 6136,
            "expected_scored_count": 6136,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 3,
      "task_id": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "domain": "Evidence & probability",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "two products, each with binary ratings from four experts of different validity",
          "original_papers": [
            {
              "citation": "Hilbig, B. E. & Moshagen, M. Generalized outcome-based strategy classification: Comparing deterministic and probabilistic choice models. Psychonomic bulletin & review 21 , 1431–1443 (2014).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib46",
              "relation": "source_study"
            }
          ],
          "input": "Evaluation matrix and expert validity order",
          "output": "two products and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; weights of high-validity cues and of the cue majority; fit of strategy predictions. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Weighted-additive model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx8",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "information on both products kept; for weighted-additive, take-the-best and others, original formal comparisons must be distinguished from newly added diagnostics",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_extended.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_extended.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "multi_attribute_choice",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 44,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "multi_attribute_choice",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 576,
            "expected_full_input_count": 576,
            "original_scored_count": 576,
            "expected_scored_count": 576,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 4,
      "task_id": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "domain": "Evidence & probability",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "event; negation; conjunction; disjunction and conditional probability queries",
          "original_papers": [
            {
              "citation": "Zhu, J.-Q., Sanborn, A. N. & Chater, N. The bayesian sampler: Generic bayesian inference causes incoherence in human probability judgments. Psychological review 127 , 719 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib122",
              "relation": "source_study"
            }
          ],
          "input": "Weather events and questions about their logical combinations",
          "output": "0–100 explicit probability estimate",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; additivity/complement/conditional coherence; human numeric-response NLL. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "This source gives no directly comparable domain baseline",
              "status": "not_located",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx2",
              "note": "Checked the official Modeling details mapping; Centaur/Llama comparisons or proposed strategies are not claimed to be the original official cognitive baselines."
            }
          ],
          "adaptation_note": "A different paradigm from CogBench's urn inference, which provides explicit priors/likelihoods; cannot be merged just because they share a name",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_four_task_responses_20261018.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "required": true,
            "choices": [
              "probability_coherence_judgment",
              "answer_self_assessment"
            ]
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "default_expression": "ROOT / 'examples/laya_bandit_choice_agent.py'"
          },
          {
            "flags": [
              "--model-label"
            ],
            "default": "Laya English"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/run_four_task_responses_20261018.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "probability_coherence_judgment",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction",
        "human_prediction"
      ],
      "result_protocols": [
        "full-original-prefix-integer-and-self-assessment-v1",
        "original_complete_integer_sequence",
        "original_scalar_numeric_readout"
      ],
      "available_metric_definitions": [
        [
          "human_nll",
          "mean_typed_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_rate",
          "invalid_responses_over_all_required_responses",
          "proportion",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "sum(invalid_response)/expected_responses",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "all_required_responses"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "mae",
          "mean_numeric_absolute_error",
          "original_human_numeric_response",
          "probability_judgment_percentage_points_0_to_100",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 79,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "probability_coherence_judgment",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 2460,
            "expected_full_input_count": 2460,
            "original_scored_count": 2460,
            "expected_scored_count": 2460,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 5,
      "task_id": "shj_category_learning",
      "display_name": "SHJ categories",
      "domain": "Concept & category learning",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "different binary-feature combinations and classification rules; rule difficulty compared after learning with feedback",
          "original_papers": [
            {
              "citation": "Badham, S. P., Sanborn, A. N. & Maylor, E. A. Deficits in category learning in older adults: Rule-based versus clustering accounts. Psychology and Aging 32 , 473 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib95",
              "relation": "source_study"
            }
          ],
          "input": "Shape, size and color descriptions, and past category feedback within the current problem",
          "output": "two categories and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; original task learning curves; accuracy per rule; condition-effect analysis to be extracted later. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Generalized context model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx1",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "The verbalized visual features are already a Psych-101 transcription, and testing concept learning is not the same as the original visual recognition",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_typed.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true",
            "help": "Audit all source rows on CPU; no model inference."
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "llama8b",
              "minitaur8b",
              "intern_decision_4b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_typed.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "category_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_choice_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ]
      ],
      "public_result_rows": 44,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "shj_category_learning",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 3664,
            "expected_full_input_count": 3664,
            "original_scored_count": 3664,
            "expected_scored_count": 3664,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 6,
      "task_id": "medin_category_learning",
      "display_name": "Medin categories",
      "domain": "Concept & category learning",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "combinations of three binary features, training classification and test typicality ratings; linear separability manipulation",
          "original_papers": [
            {
              "citation": "Levering, K. R., Conaway, N. & Kurtz, K. J. Revisiting the linear separability constraint: New implications for theories of human category learning. Memory & cognition 48 , 335–347 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib108",
              "relation": "source_study"
            }
          ],
          "input": "three-feature encoding; category feedback; phase",
          "output": "two categories and probabilities; test typicality 1–9",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; category learning/generalization; relation between typicality and category structure. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Generalized context model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx1",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Category choice and typicality ratings are kept separate; do not treat typicality as a probability",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_category.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices": [
              "medin",
              "weather"
            ],
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_category.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "medin",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "objective_label_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "mae",
          "native_rating_MAE",
          "original_human_numeric_response",
          "typicality_rating_steps_1_to_9",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 159,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "medin_category_learning",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 6972,
            "expected_full_input_count": 6972,
            "original_scored_count": 6972,
            "expected_scored_count": 6972,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 7,
      "task_id": "weather_prediction",
      "display_name": "Weather prediction",
      "domain": "Concept & category learning",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "combinations of one to three cards mapped to probabilistic sunny/rainy feedback",
          "original_papers": [
            {
              "citation": "Speekenbrink, M., Channon, S. & Shanks, D. R. Learning strategies in amnesia. Neuroscience & Biobehavioral Reviews 32 , 292–310 (2008).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib115",
              "relation": "source_study"
            }
          ],
          "input": "card combination; past weather feedback",
          "output": "Sunny/rainy and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; learning curves; sensitivity to cue combinations; generalization. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Generalized context model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx1",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "probabilistic feedback kept rather than deterministic labels; the same card combination can have different outcomes",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_category.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices": [
              "medin",
              "weather"
            ],
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_category.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "weather",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "objective_label_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ]
      ],
      "public_result_rows": 64,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "weather_prediction",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 400,
            "expected_full_input_count": 400,
            "original_scored_count": 400,
            "expected_scored_count": 400,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 8,
      "task_id": "rlwm",
      "display_name": "RLWM",
      "domain": "Learning & history",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Set size 2–6; stimulus repetition count; repetition spacing",
          "original_papers": [
            {
              "citation": "Collins & Frank 2012; the author's later WMH implementation",
              "url": "https://doi.org/10.1111/j.1460-9568.2011.07980.x",
              "relation": "source_study"
            }
          ],
          "input": "Original stimuli and visible choices/feedback",
          "output": "three actions; presentation/history protocol for neural models not yet implemented",
          "history": "Original-parameter replay is an in-sample fit; the authors' simulation copies the first human choices and cannot be treated as full autonomous action; full history is not equivalent to human working memory capacity",
          "split_protocol": "79 IDs/53,573 rows; 52,911 valid responses in the original learning analysis; no official ML split",
          "metrics": "Learning curves over set size 2–6 and presentation count; error patterns; behavioral NLL; spacing effects yet to be compiled",
          "baselines": [
            {
              "name": "original RL/WM and RLWM models; authors' WMH implementation",
              "status": "official",
              "source_url": "https://doi.org/10.1111/j.1460-9568.2011.07980.x",
              "note": "Replay of original fitted parameters for 7 models completed; not claimed as a new fit or autonomous performance"
            }
          ],
          "adaptation_note": "Original-parameter replay is an in-sample fit; the authors' simulation copies the first human choices and cannot be treated as full autonomous action; full history is not equivalent to human working memory capacity",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://osf.io/wgzb6/",
              "availability": "public",
              "note": "Original trial-by-trial data, available locally"
            },
            {
              "kind": "analysis",
              "url": "https://github.com/AnneCollins/WMH",
              "availability": "public",
              "note": "authors' later model implementation; parameter replay is not a new fit"
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_rlwm_typed.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices": [
              "rlwm"
            ],
            "default": "rlwm"
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_rlwm_typed.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "rlwm",
      "supported_unified_cli": [],
      "result_modes": [
        "original_human_history_replay_no_official_heldout_split"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "objective_label_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "same_original_human_history_response_no_official_heldout_split",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 58,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "rlwm",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 52911,
            "expected_full_input_count": 52911,
            "original_scored_count": 52911,
            "expected_scored_count": 52911,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 9,
      "task_id": "recent_probes",
      "display_name": "Recent probes",
      "domain": "Learning & history",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Membership judgment for the current six-letter set; interference from familiar probes from the previous set",
          "original_papers": [
            {
              "citation": "Enkavi, A. Z. et al. Large-scale analysis of test–retest reliabilities of self-regulation measures. Proceedings of the National Academy of Sciences 116 , 5472–5477 (2019).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib62",
              "relation": "source_study"
            }
          ],
          "input": "Current set; probe and preceding sets",
          "output": "In current set/not in set, with probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; error difference between recent and nonrecent negative probes; hits/false alarms. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rational model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx13",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "The original set disappears; persistently visible text only tests source judgment supported by external history",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_extended.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_extended.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "recent_probes",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_full_prefix_typed"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "objective_label_accuracy_under_human_history",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_agreement",
          "human_response_agreement",
          "observed_human_response_not_objective_accuracy",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "typed_choice_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 64,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "recent_probes",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 3774,
            "expected_full_input_count": 3774,
            "original_scored_count": 3774,
            "expected_scored_count": 3774,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 10,
      "task_id": "wcst",
      "display_name": "WCST",
      "domain": "Adaptation & flexibility",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Hidden sorting-rule switches; feedback valence; rule stability period",
          "original_papers": [
            {
              "citation": "Namazova et al. 2025, public cWCST data and protocol",
              "url": "https://arxiv.org/abs/2508.07887",
              "relation": "source_study"
            }
          ],
          "input": "cards; original instructions; complete human/own feedback history",
          "output": "A/B/C/D",
          "history": "Run on the original hidden-rule timeline; Laya native argmax differs from the original decoding of the two 8B models; four-option NLL and full-vocabulary NLL are reported separately",
          "split_protocol": "375 participants, original order 300 train/75 test; 12,754 test trials per mode",
          "metrics": "accuracy; perseveration error; set-loss error; human prediction NLL; participants weighted equally",
          "baselines": [
            {
              "name": "Selective Learning (SL)",
              "status": "official",
              "source_url": "https://arxiv.org/abs/2508.07887",
              "note": "Local results exist for both the original published parameters and the full refit"
            }
          ],
          "adaptation_note": "Run on the original hidden-rule timeline; Laya native argmax differs from the original decoding of the two 8B models; four-option NLL and full-vocabulary NLL are reported separately",
          "resources": []
        },
        {
          "source_library": "PsyToolkit",
          "manipulations": "Hidden color/shape/number rule switches; correct/incorrect feedback",
          "original_papers": [
            {
              "citation": "Grant, D. A., & Berg, E. (1948). A behavioral analysis of degree of reinforcement and ease of shifting to new responses in Weigl-type card-sorting problem. Journal of Experimental Psychology, 38, 404-411.",
              "url": "https://www.psytoolkit.org/experiment-library/wcst.html#_further_reading",
              "relation": "source_study"
            }
          ],
          "input": "Card to be sorted; reference cards and feedback history",
          "output": "One of four reference cards",
          "history": "action-feedback history used to infer the rule",
          "split_protocol": "The experiment toolkit has no unified ML train/test split; use the blocks, stimulus tables and exclusion rules of the selected original paper. The browser demo length does not automatically equal the formal protocol of the original study.",
          "metrics": "Accuracy; number of categories completed; perseverative errors; failures to maintain the rule",
          "baselines": [
            {
              "name": "Published model baseline not located for this library",
              "status": "not_located",
              "source_url": "https://www.psytoolkit.org/experiment-library/wcst.html",
              "note": "Demonstrated correct answers/descriptions of human effects are not comparable model baselines; these must be filled in according to the selected original paper."
            }
          ],
          "adaptation_note": "PsyToolkit explicitly calls it an inspired task, not interchangeable with the original-paper WCST protocol of this project",
          "resources": [
            {
              "kind": "environment",
              "url": "https://www.psytoolkit.org/experiment-library/wcst.html",
              "availability": "public",
              "note": "Official browser experiment instructions; demo entry point and code download link; the experiment was not run."
            },
            {
              "kind": "materials",
              "url": "https://www.psytoolkit.org/doc_exp/wcst.html",
              "availability": "public",
              "note": "The PsyToolkit code; entry point checked this round, large stimulus files not downloaded."
            },
            {
              "kind": "materials",
              "url": "https://www.psytoolkit.org/doc_exp/wcst.zip",
              "availability": "public",
              "note": "The PsyToolkit code zip file; entry point checked this round, large stimulus files not downloaded."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_wcst_models.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--model"
            ],
            "choices_expression": "MODEL_DIRS",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--smoke"
            ],
            "action": "store_true",
            "help": "Both modes, first and longest state only; not a benchmark score"
          },
          {
            "flags": [
              "--smoke-subject"
            ],
            "help": "Use the CPU-checked longest participant for smoke"
          }
        ],
        "help_command": "python scripts/run_wcst_models.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [
        "generative",
        "predictive"
      ],
      "result_protocols": [
        "wcst_generative_75_test_participants",
        "wcst_predictive_75_test_participants"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "participant_mean",
          "original_rule_correct_card",
          "fraction",
          "participant_mean"
        ],
        [
          "accuracy",
          "trial_weighted_mean",
          "original_rule_correct_card",
          "fraction",
          "trial_weighted_mean"
        ],
        [
          "event_answer_brier",
          "event_answer_brier",
          "original_ground_rule_correct_key",
          "proportion",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "mean(sum(p-onehot_ground_key)^2)",
          "original_hidden_rule_ground_key",
          "sum_over_four_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "participant_mean",
          "observed_human_response",
          "nats_per_response",
          "participant_mean"
        ],
        [
          "human_nll",
          "trial_weighted_mean",
          "observed_human_response",
          "nats_per_response",
          "trial_weighted_mean"
        ],
        [
          "human_nll_original_full_vocab",
          "participant_mean",
          "human_nll_original_full_vocab",
          "nats_per_response",
          "participant_mean"
        ],
        [
          "human_nll_original_full_vocab",
          "trial_weighted_mean",
          "human_nll_original_full_vocab",
          "nats_per_response",
          "trial_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_rate",
          "invalid_responses_over_all_required_responses",
          "proportion",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "sum(invalid_outputs)/sum(trials)",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "all_trial_denominator"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "perseveration_error",
          "participant_mean",
          "perseveration_error",
          "dimensionless",
          "participant_mean"
        ],
        [
          "perseveration_error",
          "trial_weighted_mean",
          "perseveration_error",
          "dimensionless",
          "trial_weighted_mean"
        ],
        [
          "set_loss_error",
          "participant_mean",
          "set_loss_error",
          "dimensionless",
          "participant_mean"
        ],
        [
          "set_loss_error",
          "trial_weighted_mean",
          "set_loss_error",
          "dimensionless",
          "trial_weighted_mean"
        ]
      ],
      "public_result_rows": 164,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "wcst",
            "mode": "human_predictive",
            "status": "complete",
            "full_input_count": 12754,
            "expected_full_input_count": 12754,
            "original_scored_count": 12754,
            "expected_scored_count": 12754,
            "full_end_to_end": "passed"
          },
          {
            "task": "wcst",
            "mode": "autonomous_generative",
            "status": "complete",
            "full_input_count": 12754,
            "expected_full_input_count": 12754,
            "original_scored_count": 12754,
            "expected_scored_count": 12754,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 11,
      "task_id": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "domain": "Adaptation & flexibility",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Time-varying reward; feedback history; independent confidence report",
          "original_papers": [
            {
              "citation": "CogBench ICML 2024",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Own full choice/reward history",
          "output": "F/J, then an independently reported probability that the chosen machine is currently the best",
          "history": "The independent confidence does not come from the maximum action probability; the original post hoc scoring at the boundary and the added ex ante metric are reported separately; it also supplements metacognition and is not counted as a separate task",
          "split_protocol": "10 runs×80 trials, 800 actions + 800 independent confidence reports/model",
          "metrics": "reward; accuracy; QSR; adjusted QSR; confidence; ex ante/ex post scoring reported separately",
          "baselines": [
            {
              "name": "Original paper LLM/uniform reference",
              "status": "author_reference",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "A stronger adaptive-learning cognitive controller was not located in this round"
            }
          ],
          "adaptation_note": "The independent confidence does not come from the maximum action probability; the original post hoc scoring at the boundary and the added ex ante metric are reported separately; it also supplements metacognition and is not counted as a separate task",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench",
              "availability": "public",
              "note": "Original source code is available locally; official version/scoring retained"
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Reward means of the four options drift over time; feedback after each choice",
          "original_papers": [
            {
              "citation": "Bahrami, B. & Navajas, J. 4 arm bandit task dataset (2022). URL osf.io/f3t2a .",
              "url": "https://osf.io/f3t2a",
              "relation": "source_study"
            }
          ],
          "input": "Four valid arms; past choices and scores",
          "output": "One-of-four choice, with probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; tracking the best arm; choice switching; reward; human trajectories can only score behavior prediction. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Current public transcripts provide feedback on actual actions; closed-loop reward for unexecuted arms still requires the original generating environment",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Hazard rate is stated explicitly; the two arms' means change abruptly and independently, and the change rate varies across games",
          "original_papers": [
            {
              "citation": "Xiong, H.-D., Ji-An, L., Mattar, M. & Wilson, R. Neural network modeling reveals diverse human exploration behaviors via state space analysis (2023).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib121",
              "relation": "source_study"
            }
          ],
          "input": "two options; observed rewards; hazard rate",
          "output": "two-alternative choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; adjustment after change; hazard sensitivity; exploration behavior. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Independent abrupt changes and drift/alternating reversals are kept as separate variants, and the same reward generator cannot be reused",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "two arms alternate between high and low reward distributions; confidence reported independently after each choice",
          "original_papers": [
            {
              "citation": "Ershadmanesh et al. (2023). Meta-cognitive efficiency in learned value-based choice. CCN.",
              "url": "https://doi.org/10.32470/CCN.2023.1570-0",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Observed rewards; valid arms",
          "output": "Report 0–1 confidence after the choice",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Source code defaults to 10 runs; the local original protocol has 80 trials per run; the original feedback/reward switch step order and the corrected reference are reported separately.",
          "metrics": "Reward; accuracy; QSR; mean confidence; adjusted QSR",
          "baselines": [
            {
              "name": "Uniform reference and original human performance",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "An independent confidence query is not the same as an action probability; correctness coding before/after a switch must follow the original script.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/RestlessBandit",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/RestlessBandit/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_paper_restless.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          }
        ],
        "help_command": "python scripts/run_paper_restless.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [
        "autonomous_generated_environment"
      ],
      "result_protocols": [
        "cogbench_restless_action_and_confidence_10x80",
        "documented_task_adapter"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "accuracy_post_step_original",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "mean_of_10_equal_length_80_trial_run_scores"
        ],
        [
          "accuracy",
          "accuracy_pre_step_additional",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "mean_of_10_equal_length_80_trial_run_scores"
        ],
        [
          "event_answer_brier",
          "1_minus_qsr_original",
          "independent_confidence_selected_arm_is_best_post_step_original",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "1_minus_qsr_pre_step_additional",
          "independent_confidence_selected_arm_is_best_pre_step_additional",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "event_brier_post_step_original",
          "Independently reported confidence that the chosen machine is currently best, paired with latent-mean best-arm label; never action policy probability.",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "event_brier_pre_step_additional",
          "Independently reported confidence that the chosen machine is currently best, paired with latent-mean best-arm label; never action policy probability.",
          "binary_single_event_Brier",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "original_adapter_output_invalid_rate",
          "original_legal_interface",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_two_queries_seconds",
          "two_requests_per_environment_trial",
          "seconds_per_action_plus_independent_confidence",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_two_queries_seconds",
          "two_requests_per_environment_trial",
          "seconds_per_action_plus_independent_confidence",
          "recorded_request_quantile"
        ],
        [
          "pseudo_regret",
          "pseudo_regret_pre_step_mean_additional",
          "latent_expected_reward_gap",
          "task_specific_reward_units",
          "response_weighted_mean"
        ],
        [
          "pseudo_regret",
          "pseudo_regret_pre_step_total_additional",
          "latent_expected_reward_gap",
          "task_specific_reward_units",
          "sum_over_trials"
        ],
        [
          "reward",
          "mean_reward_original",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "mean_of_10_equal_length_80_trial_run_scores"
        ]
      ],
      "public_result_rows": 64,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "nonstationary_bandit",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 1600,
            "expected_full_input_count": 1600,
            "original_scored_count": 1600,
            "expected_scored_count": 1600,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_sonnet_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 12,
      "task_id": "instrumental_learning",
      "display_name": "Instrumental learning",
      "domain": "Adaptation & flexibility",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Four contexts with two options each; probabilistic rewards and positive/negative prediction errors",
          "original_papers": [
            {
              "citation": "Lefebvre, G., Lebreton, M., Meyniel, F., Bourgeois-Gironde, S. & Palminteri, S. Behavioural and neural characterization of optimistic reinforcement learning. Nature Human Behaviour 1 , 0067 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib107",
              "relation": "source_study"
            }
          ],
          "input": "Context; two legal options and feedback history",
          "output": "two-alternative choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; learning rate; positive/negative update difference; choice accuracy. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Psych-101 and CogBench differ in reward values/sampling schemes, so their scores cannot be mixed directly under the same name",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "Interleaved two-armed tasks in four contexts; positive/negative prediction errors and different reward probabilities",
          "original_papers": [
            {
              "citation": "Lefebvre et al. (2017). Behavioural and neural characterization of optimistic reinforcement learning. Nature Human Behaviour 1:0067.",
              "url": "https://www.nature.com/articles/s41562-017-0067",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Context; past choices/feedback",
          "output": "two-armed choice and probabilities",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Default 10 runs, each run 96 trials with four interleaved contexts; follows the original environment probabilities and fit optimization.",
          "metrics": "Mean reward; RW learning rate; positive/negative learning rate difference; original official fit code",
          "baselines": [
            {
              "name": "Rescorla-Wagner single-learning-rate and dual-learning-rate models",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "Domain-model parameter fits serve cognitive phenotyping and do not automatically represent the strongest external policy; do not call it SOTA.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/InstrumentalLearning",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/InstrumentalLearning/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_cogbench_expansion.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "required": true,
            "choices_expression": "DEFAULT_RUNS"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--runs"
            ],
            "help": "Explicit source protocol; default is original code count."
          },
          {
            "flags": [
              "--version"
            ],
            "choices": [
              "1",
              "2"
            ],
            "default": "1"
          },
          {
            "flags": [
              "--base-pay"
            ],
            "default": 500
          },
          {
            "flags": [
              "--seed"
            ],
            "default": 10000
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--inference-only"
            ],
            "action": "store_true",
            "help": "Leave original CPU scores pending."
          },
          {
            "flags": [
              "--score-only"
            ],
            "action": "store_true",
            "help": "Score an existing complete trials.csv on CPU."
          }
        ],
        "help_command": "python scripts/run_cogbench_expansion.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "instrumental",
      "supported_unified_cli": [],
      "result_modes": [
        "autonomous_generated_environment"
      ],
      "result_protocols": [
        "original_environment"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "original_latent_mean_optimal_action_rate",
          "maximal_original_latent_arm_mean",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "pseudo_regret",
          "pseudo_regret_total",
          "latent_expected_reward_gap",
          "task_specific_reward_units",
          "sum_over_trials"
        ],
        [
          "reward",
          "mean_reward",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 34,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "instrumental_learning",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 960,
            "expected_full_input_count": 960,
            "original_scored_count": 960,
            "expected_scored_count": 960,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 13,
      "task_id": "horizon",
      "display_name": "Horizon",
      "domain": "Exploration & structure",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Future opportunities 1/6; forced information 3:1/2:2; initial reward difference",
          "original_papers": [
            {
              "citation": "Wilson et al. 2014, Humans use directed and random exploration to solve the explore–exploit dilemma",
              "url": "https://pubmed.ncbi.nlm.nih.gov/25347535/",
              "relation": "original"
            },
            {
              "citation": "Wilson task; the original prediction and generation protocol of Namazova et al.",
              "url": "https://arxiv.org/abs/2508.07887",
              "relation": "source_study"
            }
          ],
          "input": "Four forced observations; observed choice rewards; future opportunities",
          "output": "H/I",
          "history": "The 31 timelines are not 31 independent model replicates; the old 1000-game exploration pilot cannot replace this formal protocol",
          "split_protocol": "Prediction uses 25-participant train/6-participant test and 6,720 free actions; generation uses 31 timelines and 34,720 free actions/model",
          "metrics": "Human NLL; original horizon×trial best-arm rate/SEM; reward and paired CI; directed/random exploration effects analyzed separately",
          "baselines": [
            {
              "name": "original three-parameter RW",
              "status": "official",
              "source_url": "https://arxiv.org/abs/2508.07887",
              "note": "Authors' original fit assets/rerun still to be added"
            }
          ],
          "adaptation_note": "The 31 timelines are not 31 independent model replicates; the old 1000-game exploration pilot cannot replace this formal protocol",
          "resources": []
        },
        {
          "source_library": "Psych-101",
          "manipulations": "After four forced samples, 1 or 6 free choices; initial information equal/unequal",
          "original_papers": [
            {
              "citation": "Feng, S. F., Wang, S., Zarnescu, S. & Wilson, R. C. The dynamics of explore–exploit decisions reveal a signal-to-noise mechanism for random exploration. Scientific reports 11 , 3077 (2021).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib99",
              "relation": "source_study"
            }
          ],
          "input": "Forced samples in the current game; reward record; remaining trials",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; directed/random exploration effects in the first free choice; reward evaluated only on environment-executed trajectories. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "different study populations and task timelines kept; the Maggie three-arm version recorded as an independent structural variant within the same family",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "After four forced samples, 1 or 6 free choices; initial information equal/unequal",
          "original_papers": [
            {
              "citation": "Sadeghiyeh, H. et al. Temporal discounting correlates with directed exploration but not with random exploration. Scientific reports 10 , 4020 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib112",
              "relation": "source_study"
            }
          ],
          "input": "Forced samples in the current game; reward record; remaining trials",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; directed/random exploration effects in the first free choice; reward evaluated only on environment-executed trajectories. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "different study populations and task timelines kept; the Maggie three-arm version recorded as an independent structural variant within the same family",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "After four forced samples, 1 or 6 free choices; initial information equal/unequal",
          "original_papers": [
            {
              "citation": "Somerville, L. H. et al. Charting the expansion of strategic exploratory behavior during adolescence. Journal of experimental psychology: general 146 , 155 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib114",
              "relation": "source_study"
            }
          ],
          "input": "Forced samples in the current game; reward record; remaining trials",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; directed/random exploration effects in the first free choice; reward evaluated only on environment-executed trajectories. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "different study populations and task timelines kept; the Maggie three-arm version recorded as an independent structural variant within the same family",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "After four forced samples, 1 or 6 free choices; initial information equal/unequal",
          "original_papers": [
            {
              "citation": "Waltz, J. A., Wilson, R. C., Albrecht, M. A., Frank, M. J. & Gold, J. M. Differential effects of psychotic illness on directed and random exploration. Computational Psychiatry (Cambridge, Mass.) 4 , 18 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib118",
              "relation": "source_study"
            }
          ],
          "input": "Forced samples in the current game; reward record; remaining trials",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; directed/random exploration effects in the first free choice; reward evaluated only on environment-executed trajectories. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "different study populations and task timelines kept; the Maggie three-arm version recorded as an independent structural variant within the same family",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "After four forced samples, 1 or 6 free choices; initial information equal/unequal",
          "original_papers": [
            {
              "citation": "Wilson, R. C., Geana, A., White, J. M., Ludvig, E. A. & Cohen, J. D. Humans use directed and random exploration to solve the explore–exploit dilemma. Journal of experimental psychology: General 143 , 2074 (2014).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib23",
              "relation": "source_study"
            }
          ],
          "input": "Forced samples in the current game; reward record; remaining trials",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; directed/random exploration effects in the first free choice; reward evaluated only on environment-executed trajectories. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "different study populations and task timelines kept; the Maggie three-arm version recorded as an independent structural variant within the same family",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Horizon changed from two arms to three arms; different forced samples and future opportunities",
          "original_papers": [
            {
              "citation": "Dubois, M. & Hauser, T. U. Value-free random exploration is linked to impulsivity. Nature Communications 13 , 4542 (2022).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib27",
              "relation": "source_study"
            }
          ],
          "input": "Original task instructions; current question/available actions and the preceding real history; for sentence reading, only short sentences are input.",
          "output": "Legal options for the current stage and their probabilities; logic self-assessment as a separate numeric value.",
          "history": "Original experimental session history kept; sentence reading is handled as in the original neural analysis, with no choices invented.",
          "split_protocol": "Paper-designated OOD; cannot be used as a model tuning set. The original model fits domain parameters on the most similar training task and then transfers, and cannot be fit on the OOD test set.",
          "metrics": "Official OOD human-choice NLL; for logic items, accuracy and self-assessment error can be analyzed separately.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Structure transfer is reported separately, and three-arm task scores are not mixed directly into two-arm Horizon.",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "needs_verification",
              "note": "The paper provides per-participant transcriptions; the independent OOD data paths and permissions must be checked item by item and are not assumed to be in the training Psych-101."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "Information asymmetry of the four forced samples × subsequent 1 or 6 choices",
          "original_papers": [
            {
              "citation": "Wilson et al. (2014). Humans use directed and random exploration to solve the explore–exploit dilemma. JEP General 143:2074.",
              "url": "https://pubmed.ncbi.nlm.nih.gov/25347535/",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Forced and free choice history; current number of opportunities",
          "output": "two-armed choice and probabilities",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Source code defaults to 1000 generated games, following the original 1/6 horizon and information conditions; this differs from the current rerun of 31 human timelines.",
          "metrics": "Directed exploration; random exploration logistic regression; mean reward",
          "baselines": [
            {
              "name": "Random policy and the paper's human reference; the exploration regression is a measurement model",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "Randomness created by external sampling from probabilities is not all attributed to the model; this is a generated environment, not humans.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/HorizonTask",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/HorizonTask/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/benchmark_paper_tasks.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--predictions"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "TASKS",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*GENERATIVE_TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/benchmark_paper_tasks.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "horizon-generative",
      "supported_unified_cli": [
        {
          "key": "horizon-predictive",
          "operations": [
            "list",
            "export",
            "score",
            "predict"
          ],
          "specification": {
            "paper": "https://arxiv.org/abs/2508.07887",
            "task_family": "exploration_and_adaptation",
            "data_kind": "published_human_behavior",
            "participants": 6,
            "decisions": 6720,
            "split": "Original six-person test; 25 training participants kept separate.",
            "metric": "legal_conditioned_nll_eps1e8",
            "epsilon": 1e-08,
            "score_status": "explicit_legal_probability_adaptation",
            "original_metric": "Full-vocabulary NLL is preserved in original model runs and reported separately.",
            "history": "Original per-game prompt; previous human choices and observed rewards.",
            "source": "external/paper_matched_centaur_evaluation/horizon/data/in/test_data.csv",
            "task_state": "full_data_export_and_prediction_scoring_available"
          }
        },
        {
          "key": "horizon-generative",
          "operations": [
            "list",
            "run"
          ],
          "specification": {
            "paper": "https://arxiv.org/abs/2508.07887",
            "task_family": "exploration_and_adaptation",
            "data_kind": "original_published_generated_reward_timelines",
            "participants": 31,
            "rows": 74400,
            "decisions": 34720,
            "protocol": "Complete original participant timelines, seeds, game history and action-specific rewards.",
            "metric": "original_notebook_optimal_choice_by_horizon_and_trial",
            "score_status": "original_environment_with_external_typed_choice_adapter",
            "source": "external/paper_matched_centaur_evaluation/horizon/data/in/timeline_structure.csv",
            "task_state": "full_closed_loop_external_callback_available",
            "action_rules": [
              "native",
              "sample"
            ],
            "native_definition": "Use a supplied non-null last_native_choice; otherwise submitted-probability argmax, ties taking the first output_key. Record the actual source per decision.",
            "sample_definition": "Explicit external categorical sampling; report separately from native.",
            "adapter_contract": "Trusted local file defines create_agent(device); returned agent implements choice_probabilities(state, question, output_keys), returning normalized probabilities in key order.",
            "inference_status": "New typed-choice interface; original LLM decoder is not reproduced."
          }
        }
      ],
      "result_modes": [
        "autonomous_generated_environment",
        "autonomous_generative",
        "generative",
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "namazova_generative_31_timelines",
        "namazova_generative_31_timelines_native",
        "namazova_human_predictive_6x1120",
        "original_native_closed_loop_timing_34720"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "mean_optimal_choice_rate",
          "latent_mean_optimal_arm",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "optimal_action_rate",
          "latent_mean_optimal_arm",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "original_latent_mean_optimal_action_rate",
          "maximal_original_latent_arm_mean",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "legal_conditioned_nll_eps1e8",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "legal_nll_eps1e8",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "original_full_vocabulary_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "native_invalid_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "whole_original_decision_latency",
          "seconds",
          "all_decision_calls"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "whole_original_decision_latency",
          "seconds",
          "all_decision_calls"
        ],
        [
          "pseudo_regret",
          "latent_mean_pseudo_regret_per_free_choice",
          "maximal_original_latent_arm_mean",
          "original_reward_units_per_free_choice",
          "response_weighted_mean"
        ],
        [
          "pseudo_regret",
          "pseudo_regret_per_free_choice",
          "maximal_original_latent_arm_mean",
          "original_reward_units_per_free_choice",
          "response_weighted_mean"
        ],
        [
          "reward",
          "mean_reward_free_additional_metric",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "response_weighted_mean"
        ],
        [
          "reward",
          "mean_reward_free_additional_metric",
          "original_free_choice_reward",
          "task_specific_reward_units",
          "free_choice_mean"
        ],
        [
          "reward",
          "reward_per_free_choice",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 51,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "horizon",
            "mode": "human_predictive",
            "status": "complete",
            "full_input_count": 6720,
            "expected_full_input_count": 6720,
            "original_scored_count": 6720,
            "expected_scored_count": 6720,
            "full_end_to_end": "passed"
          },
          {
            "task": "horizon",
            "mode": "autonomous_generative",
            "status": "complete",
            "full_input_count": 34720,
            "expected_full_input_count": 34720,
            "original_scored_count": 34720,
            "expected_scored_count": 34720,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 14,
      "task_id": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "domain": "Exploration & structure",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Number of arms; reward distribution; duration; full history/statistical summary",
          "original_papers": [
            {
              "citation": "EVOLvE ICML 2025",
              "url": "https://proceedings.mlr.press/v267/nie25b.html",
              "relation": "source_study"
            }
          ],
          "input": "original prompt + RH full action history or SH cumulative statistics",
          "output": "Original valid arm name",
          "history": "Adaptation of the official code; horizon, seed and decoding differ from the paper; no overall mean across reward scales",
          "split_protocol": "FullBench MAB 16 conditions×30 trajectories; 480 games/312,000 steps, independent for each history type",
          "metrics": "Per-condition cumulative reward; pseudo-regret; RH/SH kept separate",
          "baselines": [
            {
              "name": "UCB / Greedy / Thompson sampling",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v267/nie25b.html",
              "note": "Local reference has been run, and the corrected Greedy implementation is reported separately"
            },
            {
              "name": "Algorithmic guidance and distillation methods",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v267/nie25b.html",
              "note": "The original paper's strong comparisons have still not been fully rerun"
            }
          ],
          "adaptation_note": "Adaptation of the official code; horizon, seed and decoding differ from the paper; no overall mean across reward scales",
          "resources": []
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Ten trials per game; different risk conditions such as uncertain arms and fixed-reward arms",
          "original_papers": [
            {
              "citation": "Gershman, S. J. Deconstructing the human algorithms for exploration. Cognition 173 , 34–42 (2018).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib102",
              "relation": "source_study"
            }
          ],
          "input": "two options; past rewards; progress in the current game",
          "output": "two-armed choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; effects of uncertainty and value on choice; learning curves. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rescorla-Wagner model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx5",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "not automatically merged into the same experiment as Horizon, which specifically manipulates future opportunities",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "NeuroGym",
          "manipulations": "Number of arms; reward probabilities and magnitudes",
          "original_papers": [
            {
              "citation": "Prefrontal cortex as a meta-reinforcement learning system",
              "url": "https://www.nature.com/articles/s41593-018-0147-8",
              "relation": "source_study"
            }
          ],
          "input": "Available arms and rewards/penalties seen so far",
          "output": "Arm index",
          "history": "Choice-feedback across trials",
          "split_protocol": "Generative environment, and the official core library provides no unified train/test data split; the original version's dt, period, reward, action space and termination conditions are kept; new seed/stimulus splits are an explicitly new evaluation protocol.",
          "metrics": "Cumulative reward; arm choice rate; regret requires an explicit true-reward reference",
          "baselines": [
            {
              "name": "Official trained strong baseline for this task not located",
              "status": "not_located",
              "source_url": "https://github.com/neurogym/neurogym/blob/main/neurogym/envs/native/bandit.py",
              "note": "Environment ground-truth is not a model baseline; generic training examples do not automatically constitute a paper-matched strong baseline for this task."
            }
          ],
          "adaptation_note": "unknown rewards kept; this simple Bernoulli version does not claim to be Horizon or Restless",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/neurogym/neurogym/blob/main/neurogym/envs/native/bandit.py",
              "availability": "public",
              "note": "Official current main source code; the environment was not installed or run in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_bandit_open_models.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--adapter-file"
            ],
            "help": "Trusted local create_agent(device) with choice_probabilities."
          },
          {
            "flags": [
              "--model-label"
            ],
            "default": "external_frozen_typed_choice"
          },
          {
            "flags": [
              "--history"
            ],
            "choices": [
              "raw",
              "summary"
            ],
            "default": "raw"
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--sanity-uniform"
            ],
            "action": "store_true",
            "help": "One complete original 300-step CPU episode, not a scored baseline."
          }
        ],
        "help_command": "python scripts/run_bandit_open_models.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [
        "autonomous",
        "autonomous_classical_policy",
        "autonomous_summary_history"
      ],
      "result_protocols": [
        "fullbench_original_gaussian_NIG_standard_disclosed_reimplementation",
        "fullbench_summary_history_SH",
        "official_code_fullbench_summary_history_16_conditions"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "accuracy",
          "maximal_original_latent_arm_mean",
          "proportion",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "original_latent_mean_optimal_action_rate",
          "maximal_original_latent_arm_mean",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "count(chosen_action_not_legal)/all_steps",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "all_required_steps"
        ],
        [
          "invalid_rate",
          "invalid_rate",
          "invalid_responses_over_all_required_responses",
          "proportion",
          "all_required_steps"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p50",
          "latency_p50",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "latency_p95",
          "latency_p95",
          "model_request_wall_time",
          "seconds",
          "sample_quantile"
        ],
        [
          "pseudo_regret",
          "mean_cumulative_pseudo_regret",
          "latent_expected_reward_gap",
          "condition_specific_reward_units",
          "mean_episode_cumulative_total"
        ],
        [
          "pseudo_regret",
          "mean_cumulative_pseudo_regret",
          "sum_of_latent_mean_optimal_reward_minus_chosen_reward",
          "condition_specific_reward_units",
          "mean_episode_cumulative_total"
        ],
        [
          "pseudo_regret",
          "pseudo_regret",
          "original_latent_mean_pseudo_regret",
          "original_environment_reward_units",
          "mean_episode_cumulative"
        ],
        [
          "reward",
          "mean_cumulative_reward",
          "autonomous_environment_reward",
          "condition_specific_reward_units",
          "mean_episode_cumulative_total"
        ],
        [
          "reward",
          "mean_cumulative_reward",
          "sum_of_original_environment_observed_rewards",
          "condition_specific_reward_units",
          "mean_episode_cumulative_total"
        ],
        [
          "reward",
          "reward",
          "original_cumulative_reward",
          "original_environment_reward_units",
          "mean_episode_cumulative"
        ]
      ],
      "public_result_rows": 200,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "stationary_multiarmed_bandit",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 16,
            "expected_full_input_count": 16,
            "original_scored_count": 312000,
            "expected_scored_count": 312000,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 15,
      "task_id": "structured_bandit",
      "display_name": "Structured bandit",
      "domain": "Exploration & structure",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "rewards of the eight options have a learnable structure; each game resets after ten choices",
          "original_papers": [
            {
              "citation": "Schulz, E., Franklin, N. T. & Gershman, S. J. Finding structure in multi-armed bandits. Cognitive psychology 119 , 101261 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib113",
              "relation": "source_study"
            }
          ],
          "input": "Option positions; previous rewards and the current game",
          "output": "one-of-eight choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; structure learning; generalization to unsampled options; cumulative reward. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "GP-UCB model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx12",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "specific structure conditions must follow the original labels of the five experiments, and cannot be rebuilt on our own as a generic random bandit",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_extended.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_extended.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "structured_bandit",
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ]
    },
    {
      "task_number": 16,
      "task_id": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "domain": "Information & stopping",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Left/right sampling information; continue sampling/stop available; sampling phase",
          "original_papers": [
            {
              "citation": "D’Ambrogio et al., Nature Neuroscience 2026",
              "url": "https://www.nature.com/articles/s41593-026-02342-9",
              "relation": "source_study"
            }
          ],
          "input": "Sampling/red-dot count; authors' posterior summary; position/stage",
          "output": "Sample left/right; choose left/right",
          "history": "This is currently human prediction, not a model closed loop; the original X provides posterior support; row-level CV is not participant-level held-out",
          "split_protocol": "20 participants/163,135 rows; original row-level 8 folds; 160 fold units",
          "metrics": "original subject-mean fold-averaged NLL; pooled NLL reported separately; sampling/stopping conditional behavior can be analyzed from trajectories",
          "baselines": [
            {
              "name": "Linear / UCB / Symbolic / Hybrid-ANN / end-to-end ANN",
              "status": "official",
              "source_url": "https://www.nature.com/articles/s41593-026-02342-9",
              "note": "first four completed in full locally; the asset gap for the last item is retained"
            }
          ],
          "adaptation_note": "This is currently human prediction, not a model closed loop; the original X provides posterior support; row-level CV is not participant-level held-out",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://github.com/simonedambrogio/HybridModellingProject",
              "availability": "public",
              "note": "Original task and fitting code; currently only original human actions are predicted locally"
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_hybrid_open_prediction.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--mode"
            ],
            "choices": [
              "export",
              "predict"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--export-dir"
            ]
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--uniform-sanity"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "help": "Independent held-out rows per batch; default preserves serial inference."
          }
        ],
        "help_command": "python scripts/run_hybrid_open_prediction.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_20_subjects_8fold_frozen_prediction"
      ],
      "available_metric_definitions": [
        [
          "human_nll",
          "mean_test_loss_equal_subject_fold_mean",
          "observed_human_response",
          "nats_per_response",
          "equal_subject_equal_original_fold_mean"
        ],
        [
          "human_nll",
          "pooled_action_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "original_adapter_output_invalid_rate",
          "original_legal_interface",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ]
      ],
      "public_result_rows": 29,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "hybrid_information_sampling",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 163135,
            "expected_full_input_count": 163135,
            "original_scored_count": 163135,
            "expected_scored_count": 163135,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 17,
      "task_id": "mouselab_tree",
      "display_name": "Mouselab",
      "domain": "Information & stopping",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Hidden payoffs in the tree; information revealing and stopping; information cost",
          "original_papers": [
            {
              "citation": "Callaway et al., Nature Human Behaviour 2022",
              "url": "https://www.nature.com/articles/s41562-022-01332-8",
              "relation": "source_study"
            }
          ],
          "input": "Original tree; prior; revealed nodes",
          "output": "Valid reveal or STOP",
          "history": "Human reveals involve a 3-second wait; the new model meta-MDP has cost=0 and automatically picks the best expected route; expected value is not the actual route reward; the original Pareto has no fixed seed",
          "split_protocol": "95 participants×25 trials, 20,363 actions; original five folds for each participant; autonomous model play 23,750 games/model",
          "metrics": "Human action NLL/Brier; expected value of the best route at STOP, number of reveals; original reference for the cost curve reported separately",
          "baselines": [
            {
              "name": "original 195-entry fit reference and strategy simulate/Pareto",
              "status": "official",
              "source_url": "https://www.nature.com/articles/s41562-022-01332-8",
              "note": "For the completed scope see the original summary table; different cost settings are not combined in a ranking"
            }
          ],
          "adaptation_note": "Human reveals involve a 3-second wait; the new model meta-MDP has cost=0 and automatically picks the best expected route; expected value is not the actual route reward; the original Pareto has no fixed seed",
          "resources": []
        }
      ],
      "runner": {
        "script": "scripts/benchmark_paper_tasks.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--predictions"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "TASKS",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*GENERATIVE_TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/benchmark_paper_tasks.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "mouselab-generative",
      "supported_unified_cli": [
        {
          "key": "mouselab-predictive",
          "operations": [
            "list",
            "export",
            "score",
            "predict"
          ],
          "specification": {
            "paper": "https://doi.org/10.1038/s41562-022-01332-8",
            "task_family": "information_acquisition_and_stopping",
            "data_kind": "published_human_behavior",
            "participants": 95,
            "decisions": 20363,
            "split": "Original five folds within each participant; whole trials kept together.",
            "metric": "unsmoothed_action_nll",
            "epsilon": 0.0,
            "score_status": "original_formula_with_explicit_text_interface",
            "history": "Complete revealed-node prefix in current trial; reset between trials.",
            "source": "data/mouselab_callaway2022_official/trials.json",
            "task_state": "full_data_export_and_prediction_scoring_available"
          }
        },
        {
          "key": "mouselab-generative",
          "operations": [
            "list",
            "run"
          ],
          "specification": {
            "paper": "https://doi.org/10.1038/s41562-022-01332-8",
            "task_family": "information_acquisition_and_stopping",
            "data_kind": "new_program_generated_reward_trees_not_human_trajectories",
            "participant_mdp_slots": 95,
            "trials_per_slot": 25,
            "repeats_per_trial": 10,
            "episodes": 23750,
            "environment_cost": 0,
            "protocol": "Original Exp1 meta-MDP, legal reveal frontier, own observations and rollout(policy, s).",
            "metric": "original_STOP_best_path_expected_reward_and_reveal_count",
            "score_status": "original_environment_with_explicit_text_choice_and_seeded_replay_adaptations",
            "source": "external/callaway2022_archive/planning/model",
            "task_state": "full_closed_loop_external_callback_available",
            "action_rules": [
              "native",
              "sample"
            ],
            "adapter_contract": "Trusted local file defines create_agent(device); returned object implements choice_probabilities(state, question, output_keys).",
            "seed_status": "New same-tree seeded replay; original simulate.jl was unseeded. Original individual fitted-policy simulations are not paired with these trees.",
            "stopping_semantics": "Meta-MDP automatically selects the best expected route; human subsequent route selection and realized route payoff are not measured.",
            "cost_status": "No imposed lambda or conversion of human 3-second wait into points."
          }
        }
      ],
      "result_modes": [
        "autonomous_generated_environment",
        "autonomous_generative",
        "heldout_human_history_prediction"
      ],
      "result_protocols": [
        "original_human_predictive_95x25",
        "original_metamdp_generative_23750",
        "original_metamdp_generative_23750_native"
      ],
      "available_metric_definitions": [
        [
          "human_nll",
          "mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "strict_sdk_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "unrounded_mean_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_typed_response",
          "proportion",
          "all_decision_calls"
        ],
        [
          "invalid_rate",
          "original_adapter_output_invalid_rate",
          "original_legal_interface",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "whole_original_decision_latency",
          "seconds",
          "all_decision_calls"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "whole_original_decision_latency",
          "seconds",
          "all_decision_calls"
        ],
        [
          "mean_reveals",
          "mean_reveals",
          "mean_reveals",
          "revealed_nodes_per_generated_trial",
          "response_weighted_mean"
        ],
        [
          "reward",
          "mean_terminal_expected_reward",
          "best_path_expected_value_at_STOP",
          "task_specific_reward_units",
          "episode_mean"
        ],
        [
          "reward",
          "mean_terminal_expected_reward",
          "best_path_expected_value_at_STOP",
          "task_specific_reward_units",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 49,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "mouselab_tree",
            "mode": "human_predictive",
            "status": "complete",
            "full_input_count": 20363,
            "expected_full_input_count": 20363,
            "original_scored_count": 20363,
            "expected_scored_count": 20363,
            "full_end_to_end": "passed"
          },
          {
            "task": "mouselab_tree",
            "mode": "autonomous_generative",
            "status": "complete",
            "full_input_count": 23750,
            "expected_full_input_count": 23750,
            "original_scored_count": 23750,
            "expected_scored_count": 23750,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    },
    {
      "task_number": 18,
      "task_id": "risky_choice_information_search",
      "display_name": "Risky choice information search",
      "domain": "Information & stopping",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "six lotteries; four outcomes; each cell viewed has a cost, and one may stop at any time to choose a lottery",
          "original_papers": [
            {
              "citation": "Krueger, P. M., Callaway, F., Gul, S., Griffiths, T. L. & Lieder, F. Identifying resource-rational heuristics for risky choice. Psychological Review (2024).",
              "url": "https://doi.org/10.31234/osf.io/mg7dn",
              "relation": "source_study"
            }
          ],
          "input": "Outcome probabilities; revealed payoff matrix; cost and legal actions",
          "output": "Reveal actions over lottery × outcome, or stop and choose a lottery",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; search order; number of reveals; stopping position; reward net of costs. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "This source gives no directly comparable domain baseline",
              "status": "not_located",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx2",
              "note": "Checked the official Modeling details mapping; Centaur/Llama comparisons or proposed strategies are not claimed to be the original official cognitive baselines."
            }
          ],
          "adaptation_note": "The original two-stage response can be kept as two typed choices; a synthesized joint action must be separately labeled as an interface change",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_compact.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_compact.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "risky_choice_information_search",
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ]
    },
    {
      "task_number": 19,
      "task_id": "bart",
      "display_name": "BART",
      "domain": "Risk & time preference",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Continued pumping increases reward and explosion risk; one can stop and cash out",
          "original_papers": [
            {
              "citation": "Frey, R., Pedroni, A., Mata, R., Rieskamp, J. & Hertwig, R. Risk preference shares the psychometric structure of major psychological traits. Science advances 3 , e1701381 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib61",
              "relation": "source_study"
            }
          ],
          "input": "Balloon number; current pump count and previous explosion/cash-out feedback",
          "output": "inflate/stop and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; number of pumps; adjusted pumps on unexploded balloons; adaptation to risk feedback. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Rational model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx13",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Psych-101 Frey data and the CogBench Lejuez adaptation are kept separately, without mixing protocols",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "explosion distributions of three balloon types and pump-by-pump inflate/cash-out",
          "original_papers": [
            {
              "citation": "Lejuez et al. (2002). Evaluation of a behavioral measure of risk taking: the balloon analogue risk task (BART). JEP Applied 8:75–84.",
              "url": "https://doi.org/10.1037/1076-898X.8.2.75",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Balloon type; current pump count; previous feedback",
          "output": "inflate/cash out and probabilities",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Source code defaults to 10 runs, max_pumps128/32/8 and trials15; paper B.7.2 says each type is reduced from 15 to 10, and the paper/code discrepancy must be handled explicitly rather than silently picking one.",
          "metrics": "Reward; number of pumps; adjusted risk on unexploded balloons",
          "baselines": [
            {
              "name": "Random policy and the original paper's gender-grouped mean reference",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "Rules for explosion and loss of accumulated reward kept; the maximum number of pumps cannot be treated as optimal.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/BART",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/BART/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        },
        {
          "source_library": "PsyToolkit",
          "manipulations": "Balloon type/explosion distribution; cumulative number of pumps; stopping point",
          "original_papers": [
            {
              "citation": "Lejuez, C. W., Read, J. P., Kahler, C. W., Richards, J. B., Ramsey, S. E., Stuart, G. L., …​ & Brown, R. A. (2002). Evaluation of a behavioral measure of risk taking: the Balloon Analogue Risk Task (BART). Journal of Experimental Psychology: Applied, 8(2), 75. You can find free versions like this one on academia.edu.",
              "url": "https://www.academia.edu/download/43347539/BART.pdf",
              "relation": "source_study"
            }
          ],
          "input": "Current balloon; pump outcomes so far; cumulative reward",
          "output": "Keep pumping/cash out and stop",
          "history": "Action feedback for this balloon and preceding balloons",
          "split_protocol": "The experiment toolkit has no unified ML train/test split; use the blocks, stimulus tables and exclusion rules of the selected original paper. The browser demo length does not automatically equal the formal protocol of the original study.",
          "metrics": "Adjusted mean number of pumps; explosion rate; cash-out/net reward",
          "baselines": [
            {
              "name": "Published model baseline not located for this library",
              "status": "not_located",
              "source_url": "https://www.psytoolkit.org/experiment-library/bart.html",
              "note": "Demonstrated correct answers/descriptions of human effects are not comparable model baselines; these must be filled in according to the selected original paper."
            }
          ],
          "adaptation_note": "Hidden explosion thresholds must not leak; the PsyToolkit implementation and CogBench parameters are kept separately",
          "resources": [
            {
              "kind": "environment",
              "url": "https://www.psytoolkit.org/experiment-library/bart.html",
              "availability": "public",
              "note": "Official browser experiment instructions; demo entry point and code download link; the experiment was not run."
            },
            {
              "kind": "materials",
              "url": "https://www.psytoolkit.org/doc_exp/bart.html",
              "availability": "public",
              "note": "The PsyToolkit code; entry point checked this round, large stimulus files not downloaded."
            },
            {
              "kind": "materials",
              "url": "https://www.psytoolkit.org/doc_exp/bart.zip",
              "availability": "public",
              "note": "The PsyToolkit code zip file; entry point checked this round, large stimulus files not downloaded."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_cogbench_expansion.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "required": true,
            "choices_expression": "DEFAULT_RUNS"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--runs"
            ],
            "help": "Explicit source protocol; default is original code count."
          },
          {
            "flags": [
              "--version"
            ],
            "choices": [
              "1",
              "2"
            ],
            "default": "1"
          },
          {
            "flags": [
              "--base-pay"
            ],
            "default": 500
          },
          {
            "flags": [
              "--seed"
            ],
            "default": 10000
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--inference-only"
            ],
            "action": "store_true",
            "help": "Leave original CPU scores pending."
          },
          {
            "flags": [
              "--score-only"
            ],
            "action": "store_true",
            "help": "Score an existing complete trials.csv on CPU."
          }
        ],
        "help_command": "python scripts/run_cogbench_expansion.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "bart",
      "supported_unified_cli": [],
      "result_modes": [
        "autonomous_generated_environment"
      ],
      "result_protocols": [
        "original_environment"
      ],
      "available_metric_definitions": [
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "reward",
          "mean_reward",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 24,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "bart",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 450,
            "expected_full_input_count": 450,
            "original_scored_count": 450,
            "expected_scored_count": 450,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 20,
      "task_id": "risky_choice_description",
      "display_name": "Risky choice description",
      "domain": "Risk & time preference",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Gain/loss; explicit/unknown probabilities; presence or absence of feedback",
          "original_papers": [
            {
              "citation": "Peterson, J. C., Bourgin, D. D., Agrawal, M., Reichman, D. & Griffiths, T. L. Using large-scale experiments and machine learning to discover theories of human decision-making. Science 372 , 1209–1214 (2021).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib21",
              "relation": "source_study"
            }
          ],
          "input": "descriptions of two lotteries and feedback on past outcomes for this item",
          "output": "two-alternative choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; human choice NLL; choice proportion error; risk/ambiguity condition effects. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Prospect theory model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx2",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "The original Choices13k proportion-prediction split and the Psych-101 participant held-out split are different protocols and must be scored separately",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_risk.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_SOURCES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_risk.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "choices13k",
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ]
    },
    {
      "task_number": 21,
      "task_id": "columbia_card_task",
      "display_name": "Columbia card task",
      "domain": "Risk & time preference",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Variation in gain amount, loss amount and number of loss cards; cards are turned consecutively and winnings can be cashed out at any time",
          "original_papers": [
            {
              "citation": "Frey, R., Pedroni, A., Mata, R., Rieskamp, J. & Hertwig, R. Risk preference shares the psychometric structure of major psychological traits. Science advances 3 , e1701381 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib61",
              "relation": "source_study"
            }
          ],
          "input": "Risk parameters of this round; cards turned so far and cumulative reward",
          "output": "Turn card/stop and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; stopping position; number of cards turned; sensitivity to risk parameters; environment reward computed separately. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Decision-updated reference point model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx9",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Human offline actions can only be used for fitting, and counterfactual execution requires the original card-dealing mechanism",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_compact.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_compact.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "columbia_card_task",
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ]
    },
    {
      "task_number": 22,
      "task_id": "temporal_discounting",
      "display_name": "Temporal discounting",
      "domain": "Risk & time preference",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Original studies / datasets",
          "manipulations": "Amount; delay; immediacy; each original study's conditions reported separately",
          "original_papers": [
            {
              "citation": "A large-scale dataset of choice and response-time data in intertemporal choice, Scientific Data 2026",
              "url": "https://www.nature.com/articles/s41597-026-06947-4",
              "relation": "source_study"
            }
          ],
          "input": "Smaller-sooner and larger-later amounts/delays",
          "output": "SS/LL and choice distribution",
          "history": "Original sequence per study; rows cannot be assumed to be independent for train/test",
          "split_protocol": "100 original study identifiers kept; no unified ML split across the whole library found",
          "metrics": "SS/LL choice rate; behavioral NLL; discount curve; human RT reported from the original data",
          "baselines": [
            {
              "name": "Analysis code accompanying the original database",
              "status": "author_reference",
              "source_url": "https://osf.io/3wsae/",
              "note": "not equivalent to a unified prediction SOTA"
            },
            {
              "name": "hyperbolic / exponential discounting",
              "status": "proposed",
              "source_url": "https://www.nature.com/articles/s41597-026-06947-4",
              "note": "Before integration, go back to each source to check the original fit settings"
            }
          ],
          "adaptation_note": "merged into one family with CogBench/Psych intertemporal choice, study versions kept; model latency is not fit to human RT",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://osf.io/3wsae/",
              "availability": "public",
              "note": "The paper points to CSV/RData and code, 100 studies; the full set was not downloaded in this round"
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Gain/loss; amount; immediate/delayed options; time-point shift with the same delay",
          "original_papers": [
            {
              "citation": "Ruggeri, K. et al. The globalizability of temporal discounting. Nature Human Behaviour 6 , 1386–1397 (2022).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib59",
              "relation": "source_study"
            }
          ],
          "input": "two amounts and their payout/payment times",
          "output": "two-alternative choice and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; discounting tendency; present bias; magnitude/sign effects. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Hyperbolic discounting model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx3",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "preferences have no single correct answer, and the discount rate does not automatically represent ability",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "immediate/delayed; gain/loss; two magnitudes and four bias items",
          "original_papers": [
            {
              "citation": "Ruggeri et al. (2022). The globalizability of temporal discounting. Nature Human Behaviour 6:1386–1397.",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib59",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "two amounts and times; condition branch state",
          "output": "two-alternative choice and probabilities",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Fixed adaptive items, not procedurally generated; default 1 run, base_pay500, keeping the original 10–13 response-conditional branches.",
          "metrics": "original 0–19 discount score; present bias; subadditivity and delay asymmetry",
          "baselines": [
            {
              "name": "Original human mean and uniform reference; may refer to the official Psych-101 hyperbolic discounting but cannot claim this task has been fit",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "There is no single correct preference; the original human data report only aggregate means, and usable per-participant data are not claimed to be available.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/TemporalDiscounting",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/TemporalDiscounting/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_cogbench_expansion.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "required": true,
            "choices_expression": "DEFAULT_RUNS"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--runs"
            ],
            "help": "Explicit source protocol; default is original code count."
          },
          {
            "flags": [
              "--version"
            ],
            "choices": [
              "1",
              "2"
            ],
            "default": "1"
          },
          {
            "flags": [
              "--base-pay"
            ],
            "default": 500
          },
          {
            "flags": [
              "--seed"
            ],
            "default": 10000
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--inference-only"
            ],
            "action": "store_true",
            "help": "Leave original CPU scores pending."
          },
          {
            "flags": [
              "--score-only"
            ],
            "action": "store_true",
            "help": "Score an existing complete trials.csv on CPU."
          }
        ],
        "help_command": "python scripts/run_cogbench_expansion.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "temporal",
      "supported_unified_cli": [],
      "result_modes": [
        "autonomous_generated_environment"
      ],
      "result_protocols": [
        "original_environment"
      ],
      "available_metric_definitions": [
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 24,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "temporal_discounting",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 1,
            "expected_full_input_count": 1,
            "original_scored_count": 1,
            "expected_scored_count": 1,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 23,
      "task_id": "two_step",
      "display_name": "Two-step",
      "domain": "Planning & value",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "Interaction of reward with common/rare transitions 70/30",
          "original_papers": [
            {
              "citation": "Daw et al. 2011, Model-based influences on humans’ choices and striatal prediction errors",
              "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3077926/",
              "relation": "original"
            },
            {
              "citation": "Daw paradigm; CogBench ICML 2024 implementation",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Own history; current stage",
          "output": "X/Y, then D/F or J/K",
          "history": "interaction is not sufficient evidence of an internal planning mechanism; the human archive 40×200 differs in scale from the model",
          "split_protocol": "100 runs×20 trials; 2,000 trials/4,000 actions, no train/test",
          "metrics": "reward; reward/common/interaction from the original stay OLS and their CI; run-level reward CI reported separately",
          "baselines": [
            {
              "name": "authors' historical models and human archive",
              "status": "author_reference",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "The original archive is not equivalent to a local rerun under the same protocol"
            },
            {
              "name": "model-based/model-free cognitive reference",
              "status": "proposed",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Requires a matched implementation; Uniform is only an interface reference"
            }
          ],
          "adaptation_note": "interaction is not sufficient evidence of an internal planning mechanism; the human archive 40×200 differs in scale from the model",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench",
              "availability": "public",
              "note": "Original source code is available locally; official version/scoring retained"
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Transition structure and terminal-reward changes; different studies also manipulate reward multipliers, stories and outcome types",
          "original_papers": [
            {
              "citation": "Kool, W., Cushman, F. A. & Gershman, S. J. When does model-based control pay off? PLoS computational biology 12 , e1005090 (2016).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib29",
              "relation": "source_study"
            }
          ],
          "input": "Current stage; legal actions; past transitions and rewards",
          "output": "Two-alternative choice at the current stage and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; effect of reward×transition on subsequent repeat choices; human fit and autonomous reward kept separate. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Dual-systems model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx4",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Multiple studies from the same family, but structures/rewards are not fully identical; the interaction effect alone does not prove an internal planning mechanism",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Transition structure and terminal-reward changes; different studies also manipulate reward multipliers, stories and outcome types",
          "original_papers": [
            {
              "citation": "Kool, W., Gershman, S. J. & Cushman, F. A. Cost-benefit arbitration between multiple reinforcement-learning systems. Psychological science 28 , 1321–1333 (2017).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib30",
              "relation": "source_study"
            }
          ],
          "input": "Current stage; legal actions; past transitions and rewards",
          "output": "Two-alternative choice at the current stage and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; effect of reward×transition on subsequent repeat choices; human fit and autonomous reward kept separate. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Dual-systems model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx4",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Multiple studies from the same family, but structures/rewards are not fully identical; the interaction effect alone does not prove an internal planning mechanism",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Transition structure and terminal-reward changes; different studies also manipulate reward multipliers, stories and outcome types",
          "original_papers": [
            {
              "citation": "Zorowitz, S. & Niv, Y. Data from two-step task pilots (2023). URL osf.io/2tgjd .",
              "url": "https://osf.io/2tgjd",
              "relation": "source_study"
            }
          ],
          "input": "Current stage; legal actions; past transitions and rewards",
          "output": "Two-alternative choice at the current stage and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; effect of reward×transition on subsequent repeat choices; human fit and autonomous reward kept separate. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Dual-systems model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx4",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Multiple studies from the same family, but structures/rewards are not fully identical; the interaction effect alone does not prove an internal planning mechanism",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Same two-stage structure with the story changed to magic carpet/genie; transfer to an unseen narrative",
          "original_papers": [
            {
              "citation": "Feher da Silva, C. & Hare, T. A. Humans primarily use model-based inference in the two-stage task. Nature Human Behaviour 4 , 1053–1066 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib26",
              "relation": "source_study"
            }
          ],
          "input": "Original task instructions; current question/available actions and the preceding real history; for sentence reading, only short sentences are input.",
          "output": "Legal options for the current stage and their probabilities; logic self-assessment as a separate numeric value.",
          "history": "Original experimental session history kept; sentence reading is handled as in the original neural analysis, with no choices invented.",
          "split_protocol": "Paper-designated OOD; cannot be used as a model tuning set. The original model fits domain parameters on the most similar training task and then transfers, and cannot be fit on the OOD test set.",
          "metrics": "Official OOD human-choice NLL; for logic items, accuracy and self-assessment error can be analyzed separately.",
          "baselines": [
            {
              "name": "Dual-systems model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx4",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Different stories do not add to the paradigm count; kept isolated from the training sources of the main data.",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "needs_verification",
              "note": "The paper provides per-participant transcriptions; the independent OOD data paths and permissions must be checked item by item and are not assumed to be in the training Psych-101."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "Two-stage choice with an abstract/magic-carpet story, with fMRI recordings",
          "original_papers": [
            {
              "citation": "Feher da Silva, C., Lombardi, G., Edelson, M. & Hare, T. A. Rethinking model-based and model-free influences on mental effort and striatal prediction errors. Nature Human Behaviour 7 , 956–969 (2023).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib41",
              "relation": "source_study"
            }
          ],
          "input": "Original task instructions; current question/available actions and the preceding real history; for sentence reading, only short sentences are input.",
          "output": "Legal options for the current stage and their probabilities; logic self-assessment as a separate numeric value.",
          "history": "Original experimental session history kept; sentence reading is handled as in the original neural analysis, with no choices invented.",
          "split_protocol": "Paper-designated OOD; cannot be used as a model tuning set. The original model fits domain parameters on the most similar training task and then transfers, and cannot be fit on the OOD test set.",
          "metrics": "Official OOD human-choice NLL; for logic items, accuracy and self-assessment error can be analyzed separately.",
          "baselines": [
            {
              "name": "Dual-systems model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx4",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Different stories do not add to the paradigm count; kept isolated from the training sources of the main data.",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "needs_verification",
              "note": "The paper provides per-participant transcriptions; the independent OOD data paths and permissions must be checked item by item and are not assumed to be in the training Psych-101."
            }
          ]
        },
        {
          "source_library": "CogBench",
          "manipulations": "Common/rare transitions; previous-round reward; drifting terminal values",
          "original_papers": [
            {
              "citation": "Daw et al. (2011). Model-based influences on humans choices and striatal prediction errors. Neuron 69:1204–1215.",
              "url": "https://doi.org/10.1016/j.neuron.2011.02.027",
              "relation": "source_study"
            },
            {
              "citation": "Coda-Forno et al. (2024). CogBench: a large language model walks into a psychology lab. ICML/PMLR235:9076–9108.",
              "url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "relation": "source_study"
            }
          ],
          "input": "Current stage and available actions; full transition and reward history of this run",
          "output": "Two-alternative choice at the current stage and probabilities",
          "history": "Each step keeps the original experiment information of this run up to the current point; reset boundaries follow the original code, and history-window adaptations must be labeled.",
          "split_protocol": "Default 100 runs×20 trials; differs from the original long human task and is recorded as the formal CogBench LLM protocol.",
          "metrics": "Mean reward; reward×transition interaction in the stay regression",
          "baselines": [
            {
              "name": "Uniform reference; original stay-regression diagnostic",
              "status": "official",
              "source_url": "https://proceedings.mlr.press/v235/coda-forno24a.html",
              "note": "Named as a CogBench public reference or diagnostic fit; an ordinary uniform reference is not passed off as cognitive SOTA."
            }
          ],
          "adaptation_note": "Legal actions of both stages kept; the stay interaction is not the sole proof of the internal mechanism.",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/juliancodaforno/CogBench/tree/main/Experiments/TwoStepTask",
              "availability": "public",
              "note": "The original query.py and environments corresponding to local external/CogBench have been read; no new experiments are run in this round."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/juliancodaforno/CogBench/blob/main/Experiments/TwoStepTask/store.py",
              "availability": "public",
              "note": "Original behavior/performance metric code."
            }
          ]
        },
        {
          "source_library": "NeuroGym",
          "manipulations": "Common/rare transitions; second-stage reward",
          "original_papers": [
            {
              "citation": "Model-Based Influences on Humans Choices and Striatal Prediction Errors",
              "url": "https://www.sciencedirect.com/science/article/pii/S0896627311001255",
              "relation": "source_study"
            }
          ],
          "input": "Current stage and past state-action-reward history",
          "output": "Two-alternative choice at each stage",
          "history": "Feedback across trials and stages",
          "split_protocol": "Generative environment, and the official core library provides no unified train/test data split; the original version's dt, period, reward, action space and termination conditions are kept; new seed/stimulus splits are an explicitly new evaluation protocol.",
          "metrics": "Cumulative reward; reward×transition stay interaction",
          "baselines": [
            {
              "name": "Official trained strong baseline for this task not located",
              "status": "not_located",
              "source_url": "https://github.com/neurogym/neurogym/blob/main/neurogym/envs/native/dawtwostep.py",
              "note": "Environment ground-truth is not a model baseline; generic training examples do not automatically constitute a paper-matched strong baseline for this task."
            }
          ],
          "adaptation_note": "Uses the source code's transition/reward rules; kept separately from the different CogBench two-step implementation",
          "resources": [
            {
              "kind": "environment",
              "url": "https://github.com/neurogym/neurogym/blob/main/neurogym/envs/native/dawtwostep.py",
              "availability": "public",
              "note": "Official current main source code; the environment was not installed or run in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/benchmark_paper_tasks.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--predictions"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "TASKS",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "[*GENERATIVE_TASKS, *POSTERIOR_TASKS]",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--action-rule"
            ],
            "choices": [
              "native",
              "sample"
            ],
            "default": "native"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cpu"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/benchmark_paper_tasks.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "twostep-generative",
      "supported_unified_cli": [
        {
          "key": "twostep-generative",
          "operations": [
            "list",
            "run"
          ],
          "specification": {
            "paper": "https://proceedings.mlr.press/v235/coda-forno24a.html",
            "task_family": "evidence_integration_and_planning",
            "data_kind": "new_environment_samples_from_original_public_generator",
            "runs": 100,
            "trials_per_run": 20,
            "decisions": 4000,
            "protocol": "CogBench version 1; original transition, reward drift and complete own history.",
            "metric": "original_mean_reward_and_pooled_OLS_reward_transition_stay_interaction",
            "score_status": "original_environment_with_external_typed_choice_adapter",
            "source": "external/CogBench/Experiments/TwoStepTask/query.py",
            "task_state": "full_closed_loop_external_callback_available",
            "action_rules": [
              "native",
              "sample"
            ],
            "adapter_contract": "Trusted local file defines create_agent(device), returning an agent with choice_probabilities(state, question, output_keys).",
            "seed_status": "New explicitly seeded replay protocol; original CogBench code did not fix seeds.",
            "inference_status": "Typed-choice interface is an explicit adaptation, not the original LLM decoder.",
            "human_prediction": "Not provided: published human 40 x 200 data are a different protocol."
          }
        }
      ],
      "result_modes": [
        "autonomous_generated_environment"
      ],
      "result_protocols": [
        "cogbench_generated_100x20",
        "original_CogBench_v1_environment_with_new_typed_choice_adapter"
      ],
      "available_metric_definitions": [
        [
          "invalid_rate",
          "original_adapter_output_invalid_rate",
          "original_legal_interface",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "reward",
          "mean_reward",
          "autonomous_environment_reward",
          "task_specific_reward_units",
          "response_weighted_mean"
        ],
        [
          "stay_reward_transition_interaction",
          "stay_reward_common_interaction_original_OLS",
          "stay_regressed_on_reward_times_common_transition",
          "coefficient",
          "original_OLS"
        ]
      ],
      "public_result_rows": 29,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "two_step",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 4000,
            "expected_full_input_count": 4000,
            "original_scored_count": 4000,
            "expected_scored_count": 4000,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 24,
      "task_id": "subway_hierarchical_planning",
      "display_name": "Subway hierarchical planning",
      "domain": "Planning & value",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Start/end point changes; local adjacency information and latent network community structure",
          "original_papers": [
            {
              "citation": "Tomov, M. S., Yagati, S., Kumar, A., Yang, W. & Gershman, S. J. Discovery of hierarchical representations for efficient planning. PLoS computational biology 16 , e1007594 (2020).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib117",
              "relation": "source_study"
            }
          ],
          "input": "Current location; neighboring stations; goal and explored paths",
          "output": "Valid move in one of four directions or end on arrival, with probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; path length; arrival rate; cross-subgraph planning and route choice. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "This source gives no directly comparable domain baseline",
              "status": "not_located",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx2",
              "note": "Checked the official Modeling details mapping; Centaur/Llama comparisons or proposed strategies are not claimed to be the original official cognitive baselines."
            }
          ],
          "adaptation_note": "A text adjacency graph does not test visual navigation; the environment must maintain the information boundary for unseen nodes",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_subway_original.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--experiments"
            ],
            "choices_expression": "EXPERIMENTS",
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_subway_original.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate.",
        "Original input gaps remain. Catalogued and deferred; not counted in the 24 evaluated tasks."
      ]
    },
    {
      "task_number": 25,
      "task_id": "multi_task_reward_revaluation",
      "display_name": "Multi task reward revaluation",
      "domain": "Planning & value",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "Same room transition structure, with different resource price vectors changing task value",
          "original_papers": [
            {
              "citation": "Tomov, M. S., Schulz, E. & Gershman, S. J. Multi-task reinforcement learning in humans. Nature Human Behaviour 5 , 764–773 (2021).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib93",
              "relation": "source_study"
            }
          ],
          "input": "Room; doors; resources seen; current prices",
          "output": "three doors and probabilities",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; policy transfer under new prices; reward and feature reuse. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Multi-task reinforcement learning model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx11",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Price changes are distinguished from changes in environment structure, and the full history cannot be compressed into the current net reward",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_compact.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASK_MODULES)",
            "required": true
          },
          {
            "flags": [
              "--source"
            ],
            "default_expression": "DEFAULT_SOURCE"
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--batch-size"
            ],
            "default": 1,
            "choices": [
              1
            ]
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": [],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          }
        ],
        "help_command": "python scripts/run_psych_compact.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "reward_revaluation",
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ]
    },
    {
      "task_number": 26,
      "task_id": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "domain": "Social belief & pragmatics",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Original studies / datasets",
          "manipulations": "true/false/second-order false belief; memory and reality control items",
          "original_papers": [
            {
              "citation": "Le, Boureau & Nickel 2019, Revisiting the Evaluation of Theory of Mind through Question Answering",
              "url": "https://aclanthology.org/D19-1598/",
              "relation": "source_study"
            }
          ],
          "input": "Full event story; who observed what, and the original question",
          "output": "Original single-word answer; if options are restricted to candidate locations, record it as a separate protocol",
          "history": "complete story per item, no new closed-loop environment produced",
          "split_protocol": "original train/val/test and .trace question types kept; public data is not the same as content unseen in training",
          "metrics": "Original answer accuracy; broken down by belief order, story and question type; option NLL, if added, must be labeled as an adaptation",
          "baselines": [
            {
              "name": "original tom-qa-dataset models",
              "status": "author_reference",
              "source_url": "https://github.com/facebookresearch/ToMi",
              "note": "README links the original model resources; not rerun this time"
            },
            {
              "name": "Paper's heuristic diagnostics",
              "status": "author_reference",
              "source_url": "https://aclanthology.org/D19-1598/",
              "note": "Used to diagnose shortcuts, not an added baseline"
            }
          ],
          "adaptation_note": "Original output is not a uniform MCQ; conversion to candidate options should be reported separately and cannot be claimed as the original protocol",
          "resources": [
            {
              "kind": "materials",
              "url": "https://github.com/facebookresearch/ToMi",
              "availability": "public",
              "note": "Official generator and data with .trace classification; synthetic stories, no human responses"
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_social.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASKS)",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": []
          }
        ],
        "help_command": "python scripts/run_psych_social.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "false_belief_story_qa",
      "supported_unified_cli": [],
      "result_modes": [
        "public_scenario_qa"
      ],
      "result_protocols": [
        "social-public-full-materials-typed-v1"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "actual_objective_accuracy_check",
          "original_objective_answer_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "accuracy",
          "original_answer_exact_match_mean",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "objective_answer_brier_sum_mean",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 54,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "false_belief_story_qa",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 5994,
            "expected_full_input_count": 5994,
            "original_scored_count": 5994,
            "expected_scored_count": 5994,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 27,
      "task_id": "politeness_pragmatics",
      "display_name": "Politeness",
      "domain": "Social belief & pragmatics",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "CogGym",
          "manipulations": "True evaluation; honest/kind/mean goals and utterances",
          "original_papers": [
            {
              "citation": "Yoon, E. J., Tessler, M. H., Goodman, N. D., & Frank, M. C. (2016). Talking with tact: Polite language as a balance between kindness and informativity. In Proceedings of the 38th Annual Meeting of the Cognitive Science Society (pp. 2771–2776).",
              "url": "https://escholarship.org/uc/item/6dm139m8",
              "relation": "source_study"
            }
          ],
          "input": "Evaluation scenario and speaker goal",
          "output": "Literal judgment; 1–5 true rating; goal ratings",
          "history": "three experimental roles kept separately",
          "split_protocol": "The web page keeps the experiment flow and settings; the licensed version's official item-selection mapping or ML split has not been obtained, and the formal evaluation size is not inferred from the number of web-page scenarios.",
          "metrics": [
            "Original response measures: literal acceptance; true rating and goal inference",
            "The original paper's specific statistics, fit metrics and the item selection of the formal licensed CogGym version have not yet been verified; visible responses are recorded here, without fabricating implemented metrics."
          ],
          "baselines": [
            {
              "name": "Original cognitive/computational baseline",
              "status": "not_located",
              "source_url": "https://escholarship.org/uc/item/6dm139m8",
              "note": "This round has not yet verified the original paper's baseline code and recommended settings; this does not mean the original paper has no baseline."
            }
          ],
          "adaptation_note": "Publicly visible does not mean a full reproduction of the original experiment or held-out; the original stimuli; scales and conditions must be kept. The slider ratings; coordinates; event probabilities and numeric values each have their original meaning and cannot all be treated as action probabilities or success confidence.",
          "resources": [
            {
              "kind": "materials",
              "url": "https://coggym.org/studies/yoon2016talking-exp1",
              "availability": "public",
              "note": "The public webpage allows browsing the items; instructions and response controls; webpage metadata saved this round. Browsing permission does not mean the full dataset may be redistributed; media were not played one by one."
            },
            {
              "kind": "human_data",
              "url": "https://coggym.org/dataset-access",
              "availability": "gated",
              "note": "The licensed version requires signing a research-use agreement; in this round it was not applied for, not signed and not downloaded. The web page contains no human responses."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/lance-ying/coggym/tree/main/evaluation",
              "availability": "needs_verification",
              "note": "A public scorer exists, but the licensed item selection; human responses and per-study analyses for this entry have not been obtained, so it is not claimed that a full evaluation can be run directly."
            }
          ]
        },
        {
          "source_library": "CogGym",
          "manipulations": "0–3-heart true evaluation; informational; social; joint goals",
          "original_papers": [
            {
              "citation": "Yoon, E. J., Tessler, M. H., Goodman, N. D., & Frank, M. C. (2020). Polite Speech Emerges From Competing Social Goals. Open Mind, 4, 71–87. https://doi.org/10.1162/opmi_a_00035",
              "url": "https://doi.org/10.1162/opmi_a_00035",
              "relation": "source_study"
            },
            {
              "citation": "Yoon, E. J., Tessler, M. H., Goodman, N. D., & Frank, M. C. (2020). Polite speech emerges from competing social goals.",
              "url": "https://doi.org/10.1162/opmi_a_00035",
              "relation": "source_study"
            }
          ],
          "input": "Social scenario; goal and candidate expressions",
          "output": "Binary literal truth-value choice and the speaker's utterance choice",
          "history": "original four option-order conditions kept",
          "split_protocol": "Evaluation items are fixed by public_manifest and trial_selection_map, keeping each config's experiment flow and conditions; public items are not treated as held-out, and no ML train/test split was found.",
          "metrics": [
            "Original response measures: truth-value judgments and the distribution of utterance choices",
            "CogGym public analyze: repeated responses to the same item are averaged first, normalized by response type/scale, and experiment-level Pearson R² is computed against human means; official fixed item selection; fewer than 5 scorable pairs means exclusion by default.",
            "scorer kept per query type: for choice, human-mode match/human frequency of the chosen option; for a single slider, absolute/normalized error; for multiple sliders, correlation/MSE/MAE. Before use, select according to the actual response type of that experiment."
          ],
          "baselines": [
            {
              "name": "Human-response reference",
              "status": "official",
              "source_url": "https://github.com/lance-ying/coggym/tree/main/EML/Yoon2020Polite",
              "note": "Individual and aggregate files have been parsed; this is a human reference, not a model upper bound."
            },
            {
              "name": "Original cognitive/computational models and original LLM comparison group",
              "status": "not_located",
              "source_url": "https://doi.org/10.1162/opmi_a_00035",
              "note": "Public tasks and general scoring verified; the original studies' strong models and parameters still need verification paper by paper; SOTA is not assigned based on titles."
            },
            {
              "name": "probabilistic model of polite speech",
              "status": "author_reference",
              "source_url": "https://github.com/lance-ying/coggym/blob/main/EML/Yoon2020Polite/README.md",
              "note": "The source study's README specifies the model; code parameters are yet to be checked."
            }
          ],
          "adaptation_note": "Publicly visible does not mean a full reproduction of the original experiment or held-out; the original stimuli; scales and conditions must be kept.",
          "resources": [
            {
              "kind": "materials",
              "url": "https://github.com/lance-ying/coggym/tree/main/EML/Yoon2020Polite",
              "availability": "public",
              "note": "Read config, instruction, trial and README; media assets are in the repository and were not played one by one in this round. Differences between the CogGym reconstruction and the original paper version are retained."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/lance-ying/coggym/tree/main/evaluation",
              "availability": "public",
              "note": "Read the official scorer and evaluation README, fixed trial_selection_map; no model was run."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Yoon2020Polite/exp1/human_data_ind.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Yoon2020Polite/exp1/human_data_mean.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Yoon2020Polite/exp2/human_data_ind.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Yoon2020Polite/exp2/human_data_mean.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_social.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASKS)",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": []
          }
        ],
        "help_command": "python scripts/run_psych_social.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "politeness_pragmatics",
      "supported_unified_cli": [],
      "result_modes": [
        "aggregate_human_distribution_comparison"
      ],
      "result_protocols": [
        "social-public-full-materials-typed-v1"
      ],
      "available_metric_definitions": [
        [
          "aggregate_distribution_nll",
          "human_response_weighted_mean_nll",
          "public_aggregate_human_distribution_not_individual_holdout_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 54,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "politeness_pragmatics",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 128,
            "expected_full_input_count": 128,
            "original_scored_count": 128,
            "expected_scored_count": 128,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 28,
      "task_id": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "domain": "Social belief & pragmatics",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "CogGym",
          "manipulations": "Deception; indirect speech; irony; maxims; metaphor; humor; coherence inference",
          "original_papers": [
            {
              "citation": "Hu, J., Floyd, S., Jouravlev, O., Fedorenko, E., & Gibson, E. (2023, July). A fine-grained comparison of pragmatic language understanding in humans and language models. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) (pp. 4194-4213).",
              "url": "https://arxiv.org/abs/2212.06801",
              "relation": "source_study"
            }
          ],
          "input": "Original short context passage and candidate interpretations",
          "output": "Choose the most appropriate interpretation",
          "history": "the public version has only contextual exp1; the unreleased no-context exp2 is not added automatically",
          "split_protocol": "Evaluation items are fixed by public_manifest and trial_selection_map, keeping each config's experiment flow and conditions; public items are not treated as held-out, and no ML train/test split was found.",
          "metrics": [
            "Original response measures: interpretation choice; behavior in each pragmatic condition",
            "CogGym public analyze: repeated responses to the same item are averaged first, normalized by response type/scale, and experiment-level Pearson R² is computed against human means; official fixed item selection; fewer than 5 scorable pairs means exclusion by default.",
            "scorer kept per query type: for choice, human-mode match/human frequency of the chosen option; for a single slider, absolute/normalized error; for multiple sliders, correlation/MSE/MAE. Before use, select according to the actual response type of that experiment."
          ],
          "baselines": [
            {
              "name": "Human-response reference",
              "status": "official",
              "source_url": "https://github.com/lance-ying/coggym/tree/main/EML/Hu2023Fine",
              "note": "Individual and aggregate files have been parsed; this is a human reference, not a model upper bound."
            },
            {
              "name": "Original cognitive/computational models and original LLM comparison group",
              "status": "not_located",
              "source_url": "https://arxiv.org/abs/2212.06801",
              "note": "Public tasks and general scoring verified; the original studies' strong models and parameters still need verification paper by paper; SOTA is not assigned based on titles."
            }
          ],
          "adaptation_note": "Publicly visible does not mean a full reproduction of the original experiment or held-out; the original stimuli; scales and conditions must be kept.",
          "resources": [
            {
              "kind": "materials",
              "url": "https://github.com/lance-ying/coggym/tree/main/EML/Hu2023Fine",
              "availability": "public",
              "note": "Read config, instruction, trial and README; media assets are in the repository and were not played one by one in this round. Differences between the CogGym reconstruction and the original paper version are retained."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/lance-ying/coggym/tree/main/evaluation",
              "availability": "public",
              "note": "Read the official scorer and evaluation README, fixed trial_selection_map; no model was run."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Hu2023Fine/exp1/human_data_ind.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Hu2023Fine/exp1/human_data_mean.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_social.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASKS)",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": []
          }
        ],
        "help_command": "python scripts/run_psych_social.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "pragmatic_language_interpretation",
      "supported_unified_cli": [],
      "result_modes": [
        "aggregate_human_distribution_comparison"
      ],
      "result_protocols": [
        "social-public-full-materials-typed-v1"
      ],
      "available_metric_definitions": [
        [
          "aggregate_distribution_nll",
          "human_response_weighted_mean_nll",
          "public_aggregate_human_distribution_not_individual_holdout_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ]
      ],
      "public_result_rows": 39,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "pragmatic_language_interpretation",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 170,
            "expected_full_input_count": 170,
            "original_scored_count": 170,
            "expected_scored_count": 170,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 29,
      "task_id": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "domain": "Social belief & pragmatics",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "CogGym",
          "manipulations": "Round/exact prices; context and item type",
          "original_papers": [
            {
              "citation": "Tsvilodub, P., Gandhi, K., Zhao, H., Fränken, J.-P., Franke, M., & Goodman, N. D. (2025). Non-literal Understanding of Number Words by Language Models (Version 2). arXiv. https://doi.org/10.48550/ARXIV.2502.06204",
              "url": "https://doi.org/10.48550/arxiv.2502.06204",
              "relation": "source_study"
            },
            {
              "citation": "Kao, J. T., Wu, J. Y., Bergen, L., & Goodman, N. D. (2014). Nonliteral understanding of number words. PNAS, 111(33), 12002–12007. doi:10.1073/pnas.1407479111",
              "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4143012/",
              "relation": "original"
            }
          ],
          "input": "price dialogue or item price statement",
          "output": "probabilities for each price, or of finding it expensive, reported separately",
          "history": "original Kao2014 human data; CogGym balanced item-sampling sequences are not original participant sessions",
          "split_protocol": "Evaluation items are fixed by public_manifest and trial_selection_map, keeping each config's experiment flow and conditions; public items are not treated as held-out, and no ML train/test split was found.",
          "metrics": [
            "Original response measures: price/affect probabilities for explicit events",
            "CogGym public analyze: repeated responses to the same item are averaged first, normalized by response type/scale, and experiment-level Pearson R² is computed against human means; official fixed item selection; fewer than 5 scorable pairs means exclusion by default.",
            "scorer kept per query type: for choice, human-mode match/human frequency of the chosen option; for a single slider, absolute/normalized error; for multiple sliders, correlation/MSE/MAE. Before use, select according to the actual response type of that experiment."
          ],
          "baselines": [
            {
              "name": "Human-response reference",
              "status": "official",
              "source_url": "https://github.com/lance-ying/coggym/tree/main/EML/Tsvilodub2025Nonliteral",
              "note": "Individual and aggregate files have been parsed; this is a human reference, not a model upper bound."
            },
            {
              "name": "Original cognitive/computational models and original LLM comparison group",
              "status": "not_located",
              "source_url": "https://doi.org/10.48550/arxiv.2502.06204",
              "note": "Public tasks and general scoring verified; the original studies' strong models and parameters still need verification paper by paper; SOTA is not assigned based on titles."
            },
            {
              "name": "Kao2014 pragmatic number interpretation model",
              "status": "author_reference",
              "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4143012/",
              "note": "Source of the original human data; the original paper's model infers nonliteral numbers through communicative goals, meaning and affect. The current task materials are a CogGym/Tsvilodub reconstruction, not human data regenerated from the model."
            }
          ],
          "adaptation_note": "Publicly visible does not mean a full reproduction of the original experiment or held-out; the original stimuli; scales and conditions must be kept. The slider ratings; coordinates; event probabilities and numeric values each have their original meaning and cannot all be treated as action probabilities or success confidence. The CogGym README states explicitly that part of the original Kao2014 instructions cannot be recovered; item sampling uses balanced sequences; and part of the Exp3b text was changed at the maintainers' judgment; it is not regarded as a fully verbatim replication.",
          "resources": [
            {
              "kind": "materials",
              "url": "https://github.com/lance-ying/coggym/tree/main/EML/Tsvilodub2025Nonliteral",
              "availability": "public",
              "note": "Read config, instruction, trial and README; media assets are in the repository and were not played one by one in this round. Differences between the CogGym reconstruction and the original paper version are retained."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/lance-ying/coggym/tree/main/evaluation",
              "availability": "public",
              "note": "Read the official scorer and evaluation README, fixed trial_selection_map; no model was run."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Tsvilodub2025Nonliteral/exp1/human_data_ind.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Tsvilodub2025Nonliteral/exp1/human_data_mean.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Tsvilodub2025Nonliteral/exp3b/human_data_ind.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            },
            {
              "kind": "human_data",
              "url": "https://github.com/lance-ying/coggym/blob/main/EML/Tsvilodub2025Nonliteral/exp3b/human_data_mean.json",
              "availability": "public",
              "note": "Downloaded and parsed in this round; individual records are kept separate from aggregate means."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_psych_social.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "choices_expression": "list(TASKS)",
            "required": true
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          },
          {
            "flags": [
              "--adapter-file"
            ]
          },
          {
            "flags": [
              "--model-label"
            ],
            "choices": [
              "laya",
              "intern_decision_4b",
              "llama8b",
              "minitaur8b",
              "qwen35_4b"
            ]
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--inspect-only"
            ],
            "action": "store_true"
          },
          {
            "flags": [
              "--inspect-models"
            ],
            "default": []
          }
        ],
        "help_command": "python scripts/run_psych_social.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "nonliteral_number_interpretation",
      "supported_unified_cli": [],
      "result_modes": [
        "aggregate_human_distribution_comparison",
        "original_source_scenarios"
      ],
      "result_protocols": [
        "original_public_330_items_numeric_readout",
        "social-public-full-materials-typed-v1"
      ],
      "available_metric_definitions": [
        [
          "invalid_rate",
          "invalid_response_rate",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "latency_p50",
          "latency_p50_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "latency_p95",
          "latency_p95_seconds",
          "model_request_wall_time",
          "seconds",
          "recorded_request_quantile"
        ],
        [
          "mae",
          "absolute_error_to_human_mean_mean",
          "public_human_mean_slider_judgment",
          "reported_percentage_points_0_to_100",
          "response_weighted_mean"
        ],
        [
          "mae",
          "mae",
          "original_human_native_scale_report",
          "original_response_units",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 62,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "nonliteral_number_interpretation",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 330,
            "expected_full_input_count": 330,
            "original_scored_count": 330,
            "expected_scored_count": 330,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets."
      ],
      "full_sample_api_models": [
        "claude_haiku_5_5",
        "claude_opus_5_5",
        "claude_sonnet_5_5",
        "jev_1_14"
      ]
    },
    {
      "task_number": 30,
      "task_id": "population_comparison_confidence",
      "display_name": "Population comparison confidence",
      "domain": "Metacognition",
      "scope_status": "deferred_catalogue",
      "full_sample_protocol_models": [],
      "scope_note": "Retained in the 31-task catalogue; outside this fixed first-paper evaluation scope.",
      "sources": [
        {
          "source_library": "S1CogBench existing inventory",
          "manipulations": "population-difference difficulty; correct/incorrect; independent 1–6 confidence rating",
          "original_papers": [
            {
              "citation": "Benwell et al., npj Mental Health Research 2022",
              "url": "https://www.nature.com/articles/s44184-022-00009-4",
              "relation": "source_study"
            }
          ],
          "input": "two countries",
          "output": "Left/right, then a separate 1–6 confidence report; no feedback in the formal phase",
          "history": "Fixed 2019 labels; ordinal confidence is not divided by 6 to serve as a probability; the records read so far have no item IDs, so per-item human pairings cannot be fabricated from difficulty",
          "split_protocol": "144 formal items each for A/B, 8 difficulty levels; original data from 534 participants; 473 included; no official ML split",
          "metrics": "Difficulty versus accuracy/confidence; d′, meta-d′ and M-ratio per the original statistics; a full recomputation is currently still missing",
          "baselines": [
            {
              "name": "HMeta-d / JAGS",
              "status": "author_reference",
              "source_url": "https://www.nature.com/articles/s44184-022-00009-4",
              "note": "metacognitive statistical model, not an answering-algorithm baseline"
            }
          ],
          "adaptation_note": "Fixed 2019 labels; ordinal confidence is not divided by 6 to serve as a probability; the records read so far have no item IDs, so per-item human pairings cannot be fabricated from difficulty",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://osf.io/s3cth/",
              "availability": "public",
              "note": "public download returns 429; local snapshot 366/534 files, not counted as complete"
            },
            {
              "kind": "materials",
              "url": "https://osf.io/download/kdv97/",
              "availability": "public",
              "note": "original set A, 144 stimuli"
            }
          ]
        }
      ],
      "runner": {
        "script": null,
        "exists": false,
        "arguments": []
      },
      "catalog_task_argument": null,
      "supported_unified_cli": [],
      "result_modes": [],
      "result_protocols": [],
      "available_metric_definitions": [],
      "public_result_rows": 0,
      "cpu_reference_validation": {
        "status": "not_in_current_validation_scope",
        "modes": [],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Original input gaps remain. Catalogued and deferred; not counted in the 24 evaluated tasks."
      ]
    },
    {
      "task_number": 31,
      "task_id": "performance_self_assessment",
      "display_name": "Self-assessment",
      "domain": "Metacognition",
      "scope_status": "evaluated_first_paper",
      "full_sample_protocol_models": [
        "laya",
        "intern_decision_4b",
        "llama8b",
        "minitaur8b",
        "qwen35_4b"
      ],
      "scope_note": "Each selected task-model cell has at least one accepted full-sample named protocol; other protocols can remain pending.",
      "sources": [
        {
          "source_library": "Psych-101",
          "manipulations": "twenty grammar items; before/after estimates (number correct; relative rank; difficulty)",
          "original_papers": [
            {
              "citation": "Jansen, R. A., Rafferty, A. N. & Griffiths, T. L. A rational model of the dunning–kruger effect supports insensitivity to evidence in low performers. Nature Human Behaviour 5 , 756–763 (2021).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib28",
              "relation": "source_study"
            }
          ],
          "input": "Grammar item; five options and an independent self-assessment question",
          "output": "Five-option choice per item; self-assessment value/level",
          "history": "Official natural-language trial-by-trial sessions; when predicting the current position, only the preceding real participant history is provided; a full session can reach about 32k tokens. Switching to autonomous execution requires connecting a separate feedback environment.",
          "split_protocol": "The Centaur main analysis splits each experiment by participant into 90% training; 10% test; the official Psych-101-test is used, experiment/participant IDs are kept, and rows are not randomly re-split. The training/test phases of the data's original study are reported separately from the ML held-out split.",
          "metrics": "Official shared primary metric: mean NLL on held-out human responses; item accuracy and human NLL; predicted-score bias; relation between self-assessment and actual performance. Apart from NLL, these are not-yet-implemented analyses based on the original manipulations; they do not claim to be fully implemented official scoring.",
          "baselines": [
            {
              "name": "Lookup table model",
              "status": "official",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx10.SSSx14",
              "note": "Formal domain baseline from the Centaur paper; jointly fit on training participants and tested on unseen participants. Here the paper's formulas/mappings are verified; there is no claim that it has been rerun locally."
            }
          ],
          "adaptation_note": "Self-assessed rank/difficulty is not an event probability, and probability calibration cannot be computed directly",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101",
              "availability": "public",
              "note": "Read the data card and the original prompts in the appendix; full per-participant session transcripts. This endpoint is a training resource; for the formal held-out set see Psych-101-test."
            },
            {
              "kind": "human_data",
              "url": "https://huggingface.co/datasets/marcelbinz/Psych-101-test",
              "availability": "gated",
              "note": "Official held-out participant test; HF API gated=auto, requiring login and agreement to share contact information/access conditions; the data card requires no redistribution outside the repository, license CC-BY-ND-4.0. Anonymous reading of the original README returned 401; being able to browse the web page does not mean it can be downloaded directly. The training set cannot be used as a substitute."
            },
            {
              "kind": "analysis",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "public",
              "note": "Official evaluation; cognitive models and experiment metadata; the full behavioral data were not downloaded in this round."
            }
          ]
        },
        {
          "source_library": "Psych-101",
          "manipulations": "twenty reasoning items with before/after self-ratings of expected score/percentile/difficulty",
          "original_papers": [
            {
              "citation": "Jansen, R. A., Rafferty, A. N. & Griffiths, T. L. A rational model of the dunning–kruger effect supports insensitivity to evidence in low performers. Nature Human Behaviour 5 , 756–763 (2021).",
              "url": "https://arxiv.org/html/2410.20268v3#bib.bib28",
              "relation": "source_study"
            }
          ],
          "input": "Original task instructions; current question/available actions and the preceding real history; for sentence reading, only short sentences are input.",
          "output": "Legal options for the current stage and their probabilities; logic self-assessment as a separate numeric value.",
          "history": "Original experimental session history kept; sentence reading is handled as in the original neural analysis, with no choices invented.",
          "split_protocol": "Paper-designated OOD; cannot be used as a model tuning set. The original model fits domain parameters on the most similar training task and then transfers, and cannot be fit on the OOD test set.",
          "metrics": "Official OOD human-choice NLL; for logic items, accuracy and self-assessment error can be analyzed separately.",
          "baselines": [
            {
              "name": "Domain cognitive model not located",
              "status": "not_located",
              "source_url": "https://arxiv.org/html/2410.20268v3#Sx6.SSx12.SSSx3",
              "note": "Centaur/Llama are model controls; for logic OOD, the original paper explicitly used no transferable domain model."
            }
          ],
          "adaptation_note": "Self-assessment values are not option probabilities; the original choice task and the self-assessment are recorded separately.",
          "resources": [
            {
              "kind": "human_data",
              "url": "https://github.com/marcelbinz/Llama-3.1-Centaur-70B",
              "availability": "needs_verification",
              "note": "The paper provides per-participant transcriptions; the independent OOD data paths and permissions must be checked item by item and are not assumed to be in the training Psych-101."
            }
          ]
        }
      ],
      "runner": {
        "script": "scripts/run_four_task_responses_20261018.py",
        "exists": true,
        "arguments": [
          {
            "flags": [
              "--task"
            ],
            "required": true,
            "choices": [
              "probability_coherence_judgment",
              "answer_self_assessment"
            ]
          },
          {
            "flags": [
              "--adapter-file"
            ],
            "default_expression": "ROOT / 'examples/laya_bandit_choice_agent.py'"
          },
          {
            "flags": [
              "--model-label"
            ],
            "default": "Laya English"
          },
          {
            "flags": [
              "--device"
            ],
            "default": "cuda:0"
          },
          {
            "flags": [
              "--output"
            ],
            "required": true
          }
        ],
        "help_command": "python scripts/run_four_task_responses_20261018.py --help",
        "contract": "Task-specific research runner; inspect its inputs and adapter options. Presence does not certify clean acquisition of all task assets."
      },
      "catalog_task_argument": "answer_self_assessment",
      "supported_unified_cli": [],
      "result_modes": [
        "heldout_human_history_prediction",
        "human_history_objective_scoring"
      ],
      "result_protocols": [
        "full-original-prefix-integer-and-self-assessment-v1",
        "full_original_history_objective_answer_probabilities"
      ],
      "available_metric_definitions": [
        [
          "accuracy",
          "objective_accuracy",
          "original_objective_answer_or_optimal_action_label",
          "fraction",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "event_answer_brier",
          "original_objective_answer_key",
          "proportion",
          "response_weighted_mean"
        ],
        [
          "event_answer_brier",
          "mean(sum_k(p_k-onehot_k)^2)",
          "original_objective_answer_label",
          "sum_over_classes_Brier",
          "response_weighted_mean"
        ],
        [
          "human_nll",
          "mean_typed_nll",
          "observed_human_response",
          "nats_per_response",
          "response_weighted_mean"
        ],
        [
          "invalid_rate",
          "sum(invalid_response)/expected_responses",
          "invalid_responses_over_all_required_responses",
          "fraction",
          "all_required_responses"
        ],
        [
          "latency_p50",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "sample_median"
        ],
        [
          "latency_p95",
          "complete_saved_scalar_calls",
          "model_request_wall_time",
          "seconds",
          "nearest_rank_ceil_0.95n"
        ],
        [
          "mae",
          "mean_numeric_absolute_error",
          "original_human_numeric_response",
          "difficulty_rating_0_to_10",
          "response_weighted_mean"
        ],
        [
          "mae",
          "mean_numeric_absolute_error",
          "original_human_numeric_response",
          "number_correct_0_to_20",
          "response_weighted_mean"
        ],
        [
          "mae",
          "mean_numeric_absolute_error",
          "original_human_numeric_response",
          "percentile_rank_0_to_100",
          "response_weighted_mean"
        ]
      ],
      "public_result_rows": 178,
      "cpu_reference_validation": {
        "status": "complete",
        "modes": [
          {
            "task": "performance_self_assessment",
            "mode": "primary_protocol",
            "status": "complete",
            "full_input_count": 108248,
            "expected_full_input_count": 108248,
            "original_scored_count": 108248,
            "expected_scored_count": 108248,
            "full_end_to_end": "passed"
          }
        ],
        "scope": "CPU reference input/output/original-score execution only; not the five neural checkpoints or independent gated-input acquisition."
      },
      "data_restrictions": [
        "Original provider terms and task-specific source layouts apply. No human transcripts or participant responses are included in these website assets.",
        "Psych-101 test transcripts require original-provider access and conditions; gated transcripts are not redistributed. Original logic out-of-distribution inputs are separate."
      ],
      "full_sample_api_models": [
        "jev_1_14"
      ]
    }
  ],
  "api_models": [
    "jev_1_14",
    "claude_opus_5_5",
    "claude_sonnet_5_5",
    "claude_haiku_5_5"
  ],
  "api_full_sample_modes": {
    "jev_1_14": 27,
    "claude_opus_5_5": 2,
    "claude_sonnet_5_5": 3,
    "claude_haiku_5_5": 15
  }
}