{
  "snapshot_utc": "2026-10-27T18:06:24.511077+00:00",
  "row_count": 1080,
  "counts": {
    "not_applicable": 420,
    "available_some_protocols": 624,
    "needs_protocol_resolution": 35,
    "recorded_undefined_all_invalid": 1
  },
  "scope": "Current metric availability, not a claim that every metric or additional mode is complete.",
  "rows": [
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Full-set MAE of the explicit urn posterior probability relative to analytic Bayes; the original strict q95 score is kept separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Binary independent event: the observed ball comes from the left urn; (p-y)^2, original hidden generating label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Full-set MAE of the explicit urn posterior probability relative to analytic Bayes; the original strict q95 score is kept separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Binary independent event: the observed ball comes from the left urn; (p-y)^2, original hidden generating label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Full-set MAE of the explicit urn posterior probability relative to analytic Bayes; the original strict q95 score is kept separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Binary independent event: the observed ball comes from the left urn; (p-y)^2, original hidden generating label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Full-set MAE of the explicit urn posterior probability relative to analytic Bayes; the original strict q95 score is kept separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Binary independent event: the observed ball comes from the left urn; (p-y)^2, original hidden generating label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Full-set MAE of the explicit urn posterior probability relative to analytic Bayes; the original strict q95 score is kept separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "generated_explicit_posterior",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Binary independent event: the observed ball comes from the left urn; (p-y)^2, original hidden generating label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "1",
      "task": "bayesian_urn_inference",
      "display_name": "Bayesian urn",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only responses for which the original feedback gave an objective concentration level; no ground truth is fabricated for the no-feedback phase.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE in steps relative to the original human nine-level numeric/ordinal responses, with objective feedback-level error reported separately; not treated as a continuous concentration.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the correct concentration answers from the original feedback; sum_k(p_k-onehot_k)^2, nine classes; rows without feedback have no objective label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only responses for which the original feedback gave an objective concentration level; no ground truth is fabricated for the no-feedback phase.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE in steps relative to the original human nine-level numeric/ordinal responses, with objective feedback-level error reported separately; not treated as a continuous concentration.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the correct concentration answers from the original feedback; sum_k(p_k-onehot_k)^2, nine classes; rows without feedback have no objective label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only responses for which the original feedback gave an objective concentration level; no ground truth is fabricated for the no-feedback phase.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE in steps relative to the original human nine-level numeric/ordinal responses, with objective feedback-level error reported separately; not treated as a continuous concentration.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the correct concentration answers from the original feedback; sum_k(p_k-onehot_k)^2, nine classes; rows without feedback have no objective label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only responses for which the original feedback gave an objective concentration level; no ground truth is fabricated for the no-feedback phase.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE in steps relative to the original human nine-level numeric/ordinal responses, with objective feedback-level error reported separately; not treated as a continuous concentration.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the correct concentration answers from the original feedback; sum_k(p_k-onehot_k)^2, nine classes; rows without feedback have no objective label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only responses for which the original feedback gave an objective concentration level; no ground truth is fabricated for the no-feedback phase.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_full_text;human_predictive_compact",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE in steps relative to the original human nine-level numeric/ordinal responses, with objective feedback-level error reported separately; not treated as a continuous concentration.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the correct concentration answers from the original feedback; sum_k(p_k-onehot_k)^2, nine classes; rows without feedback have no objective label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "2",
      "task": "multiple_cue_judgment",
      "display_name": "Multiple-cue judgment",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "3",
      "task": "multi_attribute_choice",
      "display_name": "Multi-attribute choice",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE of the original 0–100 probability judgments against human responses on the same scale; not called probability correctness when there is no actual event outcome.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "MAE of the original 0–100 probability judgments against human responses on the same scale; not called probability correctness when there is no actual event outcome.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE of the original 0–100 probability judgments against human responses on the same scale; not called probability correctness when there is no actual event outcome.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE of the original 0–100 probability judgments against human responses on the same scale; not called probability correctness when there is no actual event outcome.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE of the original 0–100 probability judgments against human responses on the same scale; not called probability correctness when there is no actual event outcome.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_numeric_predictive_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "4",
      "task": "probability_coherence_judgment",
      "display_name": "Probability coherence",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original binary category labels; predicted from participant history, not claimed as an autonomous learning trajectory.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct category answer for two categories; multiclass summed Brier; not the human next-action label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original binary category labels; predicted from participant history, not claimed as an autonomous learning trajectory.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct category answer for two categories; multiclass summed Brier; not the human next-action label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original binary category labels; predicted from participant history, not claimed as an autonomous learning trajectory.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct category answer for two categories; multiclass summed Brier; not the human next-action label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original binary category labels; predicted from participant history, not claimed as an autonomous learning trajectory.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct category answer for two categories; multiclass summed Brier; not the human next-action label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original binary category labels; predicted from participant history, not claimed as an autonomous learning trajectory.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_category",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct category answer for two categories; multiclass summed Brier; not the human next-action label.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "5",
      "task": "shj_category_learning",
      "display_name": "SHJ categories",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning classifications plus 252 trained test stimuli; for the latter, labels are recovered only from the same participant's previously revealed learning feedback. The 84 new test stimuli and the 336 typicality ratings have no objective labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the MAE, in rating steps, of the nine-level typicality relative to the original human ratings; typicality is not treated as category ground truth.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning categories, plus test categories for the 252 trained stimuli recovered from the same participant's earlier learning feedback; multiclass summed Brier. All 336 test classifications are kept; only the 84 new stimuli and the 336 typicality ratings have no objective ground truth.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning classifications plus 252 trained test stimuli; for the latter, labels are recovered only from the same participant's previously revealed learning feedback. The 84 new test stimuli and the 336 typicality ratings have no objective labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the MAE, in rating steps, of the nine-level typicality relative to the original human ratings; typicality is not treated as category ground truth.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning categories, plus test categories for the 252 trained stimuli recovered from the same participant's earlier learning feedback; multiclass summed Brier. All 336 test classifications are kept; only the 84 new stimuli and the 336 typicality ratings have no objective ground truth.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning classifications plus 252 trained test stimuli; for the latter, labels are recovered only from the same participant's previously revealed learning feedback. The 84 new test stimuli and the 336 typicality ratings have no objective labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the MAE, in rating steps, of the nine-level typicality relative to the original human ratings; typicality is not treated as category ground truth.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning categories, plus test categories for the 252 trained stimuli recovered from the same participant's earlier learning feedback; multiclass summed Brier. All 336 test classifications are kept; only the 84 new stimuli and the 336 typicality ratings have no objective ground truth.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning classifications plus 252 trained test stimuli; for the latter, labels are recovered only from the same participant's previously revealed learning feedback. The 84 new test stimuli and the 336 typicality ratings have no objective labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the MAE, in rating steps, of the nine-level typicality relative to the original human ratings; typicality is not treated as category ground truth.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning categories, plus test categories for the 252 trained stimuli recovered from the same participant's earlier learning feedback; multiclass summed Brier. All 336 test classifications are kept; only the 84 new stimuli and the 336 typicality ratings have no objective ground truth.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning classifications plus 252 trained test stimuli; for the latter, labels are recovered only from the same participant's previously revealed learning feedback. The 84 new test stimuli and the 336 typicality ratings have no objective labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_learning;human_predictive_test_category;human_predictive_typicality",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the MAE, in rating steps, of the nine-level typicality relative to the original human ratings; typicality is not treated as category ground truth.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "6300 learning categories, plus test categories for the 252 trained stimuli recovered from the same participant's earlier learning feedback; multiclass summed Brier. All 336 test classifications are kept; only the 84 new stimuli and the 336 typicality ratings have no objective ground truth.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "6",
      "task": "medin_category_learning",
      "display_name": "Medin categories",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual outcome of the random weather in the original implementation, not the most likely weather category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original actual weather category; two-class summed Brier; not the optimal weather arm/human preference.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual outcome of the random weather in the original implementation, not the most likely weather category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original actual weather category; two-class summed Brier; not the optimal weather arm/human preference.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual outcome of the random weather in the original implementation, not the most likely weather category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original actual weather category; two-class summed Brier; not the optimal weather arm/human preference.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual outcome of the random weather in the original implementation, not the most likely weather category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original actual weather category; two-class summed Brier; not the optimal weather arm/human preference.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual outcome of the random weather in the original implementation, not the most likely weather category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_weather",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original actual weather category; two-class summed Brier; not the optimal weather arm/human preference.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "7",
      "task": "weather_prediction",
      "display_name": "Weather prediction",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct key for the stimulus and original valid-RT filter; original history replay accuracy.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "Existing original human replay/statistical fits have no confirmed held-out next-response split; current NLL, if any, is kept as in-sample or original-data replay.",
      "exclusions": "The RLWM seven-model replay of original participant parameters or HMeta-d fits cannot be relabeled as heldout; no new split is created on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only when the correct key for the original stimulus can be verified is the key scored as the stimulus-associated answer; reported separately from human-action Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct key for the stimulus and original valid-RT filter; original history replay accuracy.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "Existing original human replay/statistical fits have no confirmed held-out next-response split; current NLL, if any, is kept as in-sample or original-data replay.",
      "exclusions": "The RLWM seven-model replay of original participant parameters or HMeta-d fits cannot be relabeled as heldout; no new split is created on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only when the correct key for the original stimulus can be verified is the key scored as the stimulus-associated answer; reported separately from human-action Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct key for the stimulus and original valid-RT filter; original history replay accuracy.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "queued",
      "applicability": "protocol_or_target_unresolved",
      "definition": "Existing original human replay/statistical fits have no confirmed held-out next-response split; current NLL, if any, is kept as in-sample or original-data replay.",
      "exclusions": "The RLWM seven-model replay of original participant parameters or HMeta-d fits cannot be relabeled as heldout; no new split is created on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only when the correct key for the original stimulus can be verified is the key scored as the stimulus-associated answer; reported separately from human-action Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct key for the stimulus and original valid-RT filter; original history replay accuracy.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "queued",
      "applicability": "protocol_or_target_unresolved",
      "definition": "Existing original human replay/statistical fits have no confirmed held-out next-response split; current NLL, if any, is kept as in-sample or original-data replay.",
      "exclusions": "The RLWM seven-model replay of original participant parameters or HMeta-d fits cannot be relabeled as heldout; no new split is created on our own.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only when the correct key for the original stimulus can be verified is the key scored as the stimulus-associated answer; reported separately from human-action Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct key for the stimulus and original valid-RT filter; original history replay accuracy.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "original_participant_history_replay_no_official_heldout_split",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "Existing original human replay/statistical fits have no confirmed held-out next-response split; current NLL, if any, is kept as in-sample or original-data replay.",
      "exclusions": "The RLWM seven-model replay of original participant parameters or HMeta-d fits cannot be relabeled as heldout; no new split is created on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only when the correct key for the original stimulus can be verified is the key scored as the stimulus-associated answer; reported separately from human-action Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "8",
      "task": "rlwm",
      "display_name": "RLWM",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Whether the probe belongs to the current set; incorrect human responses are not labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct answer for membership in the current set; two-class summed Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Whether the probe belongs to the current set; incorrect human responses are not labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct answer for membership in the current set; two-class summed Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Whether the probe belongs to the current set; incorrect human responses are not labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct answer for membership in the current set; two-class summed Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Whether the probe belongs to the current set; incorrect human responses are not labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct answer for membership in the current set; two-class summed Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Whether the probe belongs to the current set; incorrect human responses are not labels.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_memory_probe",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct answer for membership in the current set; two-class summed Brier.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "9",
      "task": "recent_probes",
      "display_name": "Recent probes",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card under the original current implicit sorting rule; predictive/generative are scored separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card answer under that trial's original rule; label definitions are listed per mode, and human choices are not used as a substitute.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "The correct card under the original current implicit sorting rule; predictive/generative are scored separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "The correct card answer under that trial's original rule; label definitions are listed per mode, and human choices are not used as a substitute.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card under the original current implicit sorting rule; predictive/generative are scored separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card answer under that trial's original rule; label definitions are listed per mode, and human choices are not used as a substitute.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card under the original current implicit sorting rule; predictive/generative are scored separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The correct card answer under that trial's original rule; label definitions are listed per mode, and human choices are not used as a substitute.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "The correct card under the original current implicit sorting rule; predictive/generative are scored separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_rule_task",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "The correct card answer under that trial's original rule; label definitions are listed per mode, and human choices are not used as a substitute.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "10",
      "task": "wcst",
      "display_name": "WCST",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The arm with the currently best latent mean; the original post-step labels and the added pre-step labels are reported separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_choice_plus_independent_confidence",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Confidence in chosen-current-best is queried independently, and 1-QSR of the original QSR is the binary Brier; post/pre-step reported separately; the wording constraint zero = guess is kept.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The arm with the currently best latent mean; the original post-step labels and the added pre-step labels are reported separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_choice_plus_independent_confidence",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Confidence in chosen-current-best is queried independently, and 1-QSR of the original QSR is the binary Brier; post/pre-step reported separately; the wording constraint zero = guess is kept.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The arm with the currently best latent mean; the original post-step labels and the added pre-step labels are reported separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_choice_plus_independent_confidence",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Confidence in chosen-current-best is queried independently, and 1-QSR of the original QSR is the binary Brier; post/pre-step reported separately; the wording constraint zero = guess is kept.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The arm with the currently best latent mean; the original post-step labels and the added pre-step labels are reported separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_choice_plus_independent_confidence",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Confidence in chosen-current-best is queried independently, and 1-QSR of the original QSR is the binary Brier; post/pre-step reported separately; the wording constraint zero = guess is kept.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The arm with the currently best latent mean; the original post-step labels and the added pre-step labels are reported separately.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_choice_plus_independent_confidence",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Confidence in chosen-current-best is queried independently, and 1-QSR of the original QSR is the binary Brier; post/pre-step reported separately; the wording constraint zero = guess is kept.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "11",
      "task": "nonstationary_bandit",
      "display_name": "Restless bandit",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "If an optimal-arm choice rate is reported, it must use the latent reward rate of that original context and state its definition explicitly; it is not the stochastic realized reward 0/1.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "If an optimal-arm choice rate is reported, it must use the latent reward rate of that original context and state its definition explicitly; it is not the stochastic realized reward 0/1.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "If an optimal-arm choice rate is reported, it must use the latent reward rate of that original context and state its definition explicitly; it is not the stochastic realized reward 0/1.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "If an optimal-arm choice rate is reported, it must use the latent reward rate of that original context and state its definition explicitly; it is not the stochastic realized reward 0/1.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "If an optimal-arm choice rate is reported, it must use the latent reward rate of that original context and state its definition explicitly; it is not the stochastic realized reward 0/1.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_instrumental",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "12",
      "task": "instrumental_learning",
      "display_name": "Instrumental learning",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The original latent-mean optimal-choice rate on autonomous trajectories; agreement rate in human prediction is not counted.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The original latent-mean optimal-choice rate on autonomous trajectories; agreement rate in human prediction is not counted.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The original latent-mean optimal-choice rate on autonomous trajectories; agreement rate in human prediction is not counted.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The original latent-mean optimal-choice rate on autonomous trajectories; agreement rate in human prediction is not counted.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "The original latent-mean optimal-choice rate on autonomous trajectories; agreement rate in human prediction is not counted.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_published_timelines",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "13",
      "task": "horizon",
      "display_name": "Horizon",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Optimal-arm choice rate only where the original latent means exist, treated side by side according to the environment definition; not the realized reward rate.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Optimal-arm choice rate only where the original latent means exist, treated side by side according to the environment definition; not the realized reward rate.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "running",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Optimal-arm choice rate only where the original latent means exist, treated side by side according to the environment definition; not the realized reward rate.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Optimal-arm choice rate only where the original latent means exist, treated side by side according to the environment definition; not the realized reward rate.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Optimal-arm choice rate only where the original latent means exist, treated side by side according to the environment definition; not the realized reward rate.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Maximum of the original hidden means minus the mean of the chosen action; read only for scoring; specify pre-step/free action/context explicitly; if the original reference is undefined, keep it as missing.",
      "exclusions": "Not the gap to a Bayes-optimal policy with the same information, and not inferred from human NLL.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "running",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_SH;autonomous_RH",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "14",
      "task": "stationary_multiarmed_bandit",
      "display_name": "Stationary bandits",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive_original_subject_CV",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "16",
      "task": "hybrid_information_sampling",
      "display_name": "Hybrid sampling",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Current cost=0; expected value of automatically choosing the best path at STOP; the original information time-cost reference is reported separately and is not called the realized path reward.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Current cost=0; expected value of automatically choosing the best path at STOP; the original information time-cost reference is reported separately and is not called the realized path reward.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Current cost=0; expected value of automatically choosing the best path at STOP; the original information time-cost reference is reported separately and is not called the realized path reward.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Current cost=0; expected value of automatically choosing the best path at STOP; the original information time-cost reference is reported separately and is not called the realized path reward.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Current cost=0; expected value of automatically choosing the best path at STOP; the original information time-cost reference is reported separately and is not called the realized path reward.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "human_predictive;autonomous_cost0_stop_path_value",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "17",
      "task": "mouselab_tree",
      "display_name": "Mouselab",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "19",
      "task": "bart",
      "display_name": "BART",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_adaptive_preference_questionnaire",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "22",
      "task": "temporal_discounting",
      "display_name": "Temporal discounting",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Actual reward/cost units of the original autonomous environment, kept per source and mode; cannot be averaged across tasks.",
      "exclusions": "Not applicable to human-prediction modes; the original humans' reward cannot be attributed to the model.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "needs_protocol_resolution",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "protocol_or_target_unresolved",
      "definition": "The CogBench/environment modes run so far have no human next response; the catalog also has human sources such as Psych-101, whose formal modes/source denominators have not yet been confirmed.",
      "exclusions": "Unresolved human-source items are kept; N/A for the generation mode is not extended into a permanent N/A for the whole task, and sources are not added/removed on our own.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "autonomous_CogBench",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "23",
      "task": "two_step",
      "display_name": "Two-step",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct answer to the story; the visible-container option adaptation is reported separately from the original unrestricted-QA.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct location/answer label; multiclass summed Brier, keeping the actual number of categories and the option adaptation.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct answer to the story; the visible-container option adaptation is reported separately from the original unrestricted-QA.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct location/answer label; multiclass summed Brier, keeping the actual number of categories and the option adaptation.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct answer to the story; the visible-container option adaptation is reported separately from the original unrestricted-QA.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct location/answer label; multiclass summed Brier, keeping the actual number of categories and the option adaptation.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct answer to the story; the visible-container option adaptation is reported separately from the original unrestricted-QA.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Original correct location/answer label; multiclass summed Brier, keeping the actual number of categories and the option adaptation.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct answer to the story; the visible-container option adaptation is reported separately from the original unrestricted-QA.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "objective_story_question_answering",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Original correct location/answer label; multiclass summed Brier, keeping the actual number of categories and the option adaptation.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "26",
      "task": "false_belief_story_qa",
      "display_name": "False-belief QA",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "27",
      "task": "politeness_pragmatics",
      "display_name": "Politeness",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses have no numeric target defined on the original scale; option IDs and preference labels are not treated as continuous quantities.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "28",
      "task": "pragmatic_language_interpretation",
      "display_name": "Pragmatic interpretation",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE against the original numeric/multi-slider human distribution, keeping the scale/normalization rule and the original aggregation unit.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE against the original numeric/multi-slider human distribution, keeping the scale/normalization rule and the original aggregation unit.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE against the original numeric/multi-slider human distribution, keeping the scale/normalization rule and the original aggregation unit.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "recorded_undefined_all_invalid",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE against the original numeric/multi-slider human distribution, keeping the scale/normalization rule and the original aggregation unit.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current responses are preferences, information-acquisition actions or numeric reports, with no objective discrete correct answer defined by this protocol; human agreement rate is not counted as accuracy.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE against the original numeric/multi-slider human distribution, keeping the scale/normalization rule and the original aggregation unit.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no official held-out human next-response target; aggregate distributions, author archives, generated environments and in-sample fits are reported separately.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current outputs are policy/human-preference probabilities, with no independent event report or objective answer label; action Brier cannot substitute for them.",
      "exclusions": "aggregate_human_numeric_distribution_comparison_public_scenarios",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "29",
      "task": "nonliteral_number_interpretation",
      "display_name": "Nonliteral number",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only answers to objective grammar/logic items; the self-assessment numbers themselves have no correct category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "MAE is counted only when the original self-assessment has an explicit numeric scale; error relative to the human report and relative to the actual score are named separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Only the answer distribution and correct labels for objective grammar/logic items; self-assessment values are not event probabilities.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "laya",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "complete_at_least_one_full_protocol",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Only answers to objective grammar/logic items; the self-assessment numbers themselves have no correct category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "running",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "MAE is counted only when the original self-assessment has an explicit numeric scale; error relative to the human report and relative to the actual score are named separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Only the answer distribution and correct labels for objective grammar/logic items; self-assessment values are not event probabilities.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "intern_decision_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "running",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only answers to objective grammar/logic items; the self-assessment numbers themselves have no correct category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE is counted only when the original self-assessment has an explicit numeric scale; error relative to the human report and relative to the actual score are named separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only the answer distribution and correct labels for objective grammar/logic items; self-assessment values are not event probabilities.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "llama8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only answers to objective grammar/logic items; the self-assessment numbers themselves have no correct category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE is counted only when the original self-assessment has an explicit numeric scale; error relative to the human report and relative to the actual score are named separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only the answer distribution and correct labels for objective grammar/logic items; self-assessment values are not event probabilities.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "minitaur8b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "True"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "accuracy",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only answers to objective grammar/logic items; the self-assessment numbers themselves have no correct category.",
      "exclusions": "Excludes human-choice agreement, preference ratings and self-assessment levels; only the response subset under this definition is counted.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "reward",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current human-prediction/QA/numeric protocols do not execute an environment; the original humans' earnings cannot be treated as the model's autonomous reward.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "pseudo_regret",
      "closeout_status": "not_applicable",
      "run_status": "queued",
      "applicability": "not_applicable_current_protocol",
      "definition": "The current protocol has no defined and readable per-decision latent expected-reward reference; regret is not constructed from action agreement rate.",
      "exclusions": "human_predictive_grammar_and_logic;objective_answers;self_assessment_pending_interface",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "mae",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "MAE is counted only when the original self-assessment has an explicit numeric scale; error relative to the human report and relative to the actual score are named separately.",
      "exclusions": "Category ID differences are not treated as errors; ratings are not simply converted into event probabilities.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "human_nll",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Official participant/fold splits and the complete true history kept; original full-vocabulary NLL and typed legal-response NLL reported separately.",
      "exclusions": "Human predictions are not derived from autonomous trajectories; same-participant fits and aggregate human-distribution NLL are not mixed in.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "event_answer_brier",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Only the answer distribution and correct labels for objective grammar/logic items; self-assessment values are not event probabilities.",
      "exclusions": "Policy action p is not treated as success confidence; multi-class summed and binary single-event scores, and different numbers of classes, are reported separately and not averaged across tasks.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "latency_p50",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "latency_p95",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Quantiles of measured time per request or per complete decision, stating the timing scope, batch, device sharing, warmup and input; reported separately for each mode.",
      "exclusions": "Total duration/throughput cannot be used to back out per-request quantiles; batch wall-clock time is not apportioned into true per-request latency; not treated as human RT.",
      "current_full_protocol_cell": "False"
    },
    {
      "task_number": "31",
      "task": "performance_self_assessment",
      "display_name": "Self-assessment",
      "model": "qwen35_4b",
      "metric": "invalid_rate",
      "closeout_status": "available_some_protocols",
      "run_status": "queued",
      "applicability": "applicable",
      "definition": "Number of invalid responses, as defined by the original parsing/valid-option/probability-range rules, divided by the total number of responses expected under that protocol; failed and unfinished protocols are reported separately.",
      "exclusions": "Valid does not mean correct; invalid rows cannot be deleted and the valid rows then used as the full denominator.",
      "current_full_protocol_cell": "False"
    }
  ]
}