{
  "assurance": {
    "conducted_by": "Codex research assistant using deterministic counting code",
    "formal_platform_quality_assessment": false,
    "human_reviewed": false,
    "independence_certified": false,
    "published_or_imported_to_site": false
  },
  "attempt_type": "Reproduction of published dataset composition statistics; not model-performance reproduction",
  "checks": [
    {
      "basis": "arXiv:2308.11462v1 Table 7; n is interpreted as the evaluation/test slice size",
      "expected": {
        "arbitrary": 18,
        "descriptive": 19,
        "fanciful": 19,
        "generic": 19,
        "suggestive": 20
      },
      "id": "paper_table7_class_counts",
      "observed": {
        "arbitrary": 19,
        "descriptive": 19,
        "fanciful": 19,
        "generic": 19,
        "suggestive": 19
      },
      "outcome": "does_not_match"
    },
    {
      "basis": "Sum of the five n values in Table 7",
      "expected": 95,
      "id": "paper_table7_total",
      "observed": 95,
      "outcome": "match"
    },
    {
      "basis": "Pinned README Size (samples)",
      "expected": 99,
      "id": "readme_declared_total",
      "observed": 100,
      "outcome": "does_not_match"
    },
    {
      "basis": "Pinned README dataset-construction description",
      "expected": {
        "arbitrary": 1,
        "descriptive": 1,
        "fanciful": 1,
        "generic": 1,
        "suggestive": 1
      },
      "id": "readme_construction_train",
      "observed": {
        "arbitrary": 1,
        "descriptive": 1,
        "fanciful": 1,
        "generic": 1,
        "suggestive": 1
      },
      "outcome": "match"
    },
    {
      "basis": "Pinned README dataset-construction description",
      "expected": {
        "arbitrary": 19,
        "descriptive": 19,
        "fanciful": 19,
        "generic": 19,
        "suggestive": 19
      },
      "id": "readme_construction_test",
      "observed": {
        "arbitrary": 19,
        "descriptive": 19,
        "fanciful": 19,
        "generic": 19,
        "suggestive": 19
      },
      "outcome": "match"
    },
    {
      "basis": "Pinned README dataset-construction description",
      "expected": 95,
      "id": "readme_construction_test_total",
      "observed": 95,
      "outcome": "match"
    }
  ],
  "cost": {
    "additional_paid_service_cost_usd": 0,
    "input_download_bytes": 532943,
    "local_hardware_energy_cost": "not measured",
    "orchestrating_assistant_tokens_and_cost": "not measured",
    "paid_model_api_calls": 0
  },
  "dataset_provenance": {
    "attribution": "Neel Guha; LegalBench contributors, Guha et al. (2023), arXiv:2308.11462v1",
    "github_commit": "b46bf4ffae90524b2b72aaa30e7745fe9db64481",
    "github_commit_date": "2026-03-30T05:23:33Z",
    "github_repository": "https://github.com/HazyResearch/legalbench",
    "huggingface_commit": "daec8237410aa23e3faf4bc41ad8b3a7e1696826",
    "huggingface_repository": "https://huggingface.co/datasets/nguha/legalbench",
    "inputs": [
      {
        "bytes": 2787,
        "content_type": "text/plain; charset=utf-8",
        "etag": "\"542259da92ce3cb479fd5bac9ee1973c084dd6a361442c206b5491d2b6c20d63\"",
        "http_status": 200,
        "path": "inputs/task-readme.md",
        "requested_at_utc": "2026-10-02T21:00:06.680437+00:00",
        "resolved_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/tasks/abercrombie/README.MD",
        "sha256": "d78004408a522c8ec2a796a30ac3595229ceb684fa06856ed828d512d943bdae",
        "source_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/tasks/abercrombie/README.MD"
      },
      {
        "bytes": 13230,
        "content_type": "text/plain; charset=utf-8",
        "etag": "\"1b313656ddd22611eb3bf8ff63d6e261d799cd65b74240b72cd3724037fb3dde\"",
        "http_status": 200,
        "path": "inputs/upstream-evaluation.py",
        "requested_at_utc": "2026-10-02T21:00:06.745960+00:00",
        "resolved_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/evaluation.py",
        "sha256": "7bbae257d64ea9f29d97a25d92bbfca4343f3d2d495fd5c724f75b3ab0671e1a",
        "source_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/evaluation.py"
      },
      {
        "bytes": 300,
        "content_type": "text/plain; charset=utf-8",
        "etag": "\"1d778897f265d198f0c61b8a59b3601b5b52bf3193d9ea949f2dd66c5b4b84cb\"",
        "http_status": 200,
        "path": "inputs/github-train.tsv",
        "requested_at_utc": "2026-10-02T21:00:06.964132+00:00",
        "resolved_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/tasks/abercrombie/train.tsv",
        "sha256": "e06e9890ef74f5ecf84e8052c92c12311cd37451a3048bfa08e517d4adf419e9",
        "source_url": "https://raw.githubusercontent.com/HazyResearch/legalbench/b46bf4ffae90524b2b72aaa30e7745fe9db64481/tasks/abercrombie/train.tsv"
      },
      {
        "bytes": 300,
        "content_type": "text/plain; charset=utf-8",
        "etag": "\"64a4327695e04c04070d6e1e06b032078006b586\"",
        "http_status": 200,
        "path": "inputs/hf-train.tsv",
        "requested_at_utc": "2026-10-02T21:00:07.204877+00:00",
        "resolved_url": "https://huggingface.co/api/resolve-cache/datasets/nguha/legalbench/daec8237410aa23e3faf4bc41ad8b3a7e1696826/data%2Fabercrombie%2Ftrain.tsv?%2Fdatasets%2Fnguha%2Flegalbench%2Fresolve%2Fdaec8237410aa23e3faf4bc41ad8b3a7e1696826%2Fdata%2Fabercrombie%2Ftrain.tsv=&etag=%2264a4327695e04c04070d6e1e06b032078006b586%22",
        "sha256": "e06e9890ef74f5ecf84e8052c92c12311cd37451a3048bfa08e517d4adf419e9",
        "source_url": "https://huggingface.co/datasets/nguha/legalbench/resolve/daec8237410aa23e3faf4bc41ad8b3a7e1696826/data/abercrombie/train.tsv"
      },
      {
        "bytes": 5771,
        "content_type": "text/plain; charset=utf-8",
        "etag": "\"e85237f2aa4b4c798b45115729f1b14478ffe82c\"",
        "http_status": 200,
        "path": "inputs/hf-test.tsv",
        "requested_at_utc": "2026-10-02T21:00:07.421338+00:00",
        "resolved_url": "https://huggingface.co/api/resolve-cache/datasets/nguha/legalbench/daec8237410aa23e3faf4bc41ad8b3a7e1696826/data%2Fabercrombie%2Ftest.tsv?%2Fdatasets%2Fnguha%2Flegalbench%2Fresolve%2Fdaec8237410aa23e3faf4bc41ad8b3a7e1696826%2Fdata%2Fabercrombie%2Ftest.tsv=&etag=%22e85237f2aa4b4c798b45115729f1b14478ffe82c%22",
        "sha256": "8e2e9051584fa6da66ba14a6a65310b03053af39869a5673ee43a9e378debab8",
        "source_url": "https://huggingface.co/datasets/nguha/legalbench/resolve/daec8237410aa23e3faf4bc41ad8b3a7e1696826/data/abercrombie/test.tsv"
      },
      {
        "bytes": 441843,
        "content_type": "text/html; charset=utf-8",
        "etag": "\"CI2ijKKTupYDEAE=\"",
        "http_status": 200,
        "path": "inputs/paper-v1.html",
        "requested_at_utc": "2026-10-02T21:00:07.717581+00:00",
        "resolved_url": "https://arxiv.org/html/2308.11462v1",
        "sha256": "e272c2ca5946697211584ac6cecad41de7510dc98335631468ba7bbafee554c3",
        "source_url": "https://arxiv.org/html/2308.11462v1"
      },
      {
        "bytes": 9033,
        "content_type": "application/json; charset=utf-8",
        "etag": "W/\"297b77742ed877d3bd4299ebdfa70f49d479652d5a5d780daf67ba7dc1340b48\"",
        "http_status": 200,
        "path": "inputs/github-commit.json",
        "requested_at_utc": "2026-10-02T21:00:07.819250+00:00",
        "resolved_url": "https://api.github.com/repos/HazyResearch/legalbench/commits/b46bf4ffae90524b2b72aaa30e7745fe9db64481",
        "sha256": "297b77742ed877d3bd4299ebdfa70f49d479652d5a5d780daf67ba7dc1340b48",
        "source_url": "https://api.github.com/repos/HazyResearch/legalbench/commits/b46bf4ffae90524b2b72aaa30e7745fe9db64481"
      },
      {
        "bytes": 59679,
        "content_type": "application/json; charset=utf-8",
        "etag": "W/\"e91f-F0O/rB4qGdkP+t8tHhaA6p3YG70\"",
        "http_status": 200,
        "path": "inputs/hf-revision.json",
        "requested_at_utc": "2026-10-02T21:00:08.286883+00:00",
        "resolved_url": "https://huggingface.co/api/datasets/nguha/legalbench/revision/daec8237410aa23e3faf4bc41ad8b3a7e1696826",
        "sha256": "16828bc75444da68d17afa2899106fcdecb693f23ddbb48abefcb18db6b47b91",
        "source_url": "https://huggingface.co/api/datasets/nguha/legalbench/revision/daec8237410aa23e3faf4bc41ad8b3a7e1696826"
      }
    ],
    "license": "CC BY 4.0, as stated in pinned task README and paper Table 8",
    "license_url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "execution": {
    "command": "python3 -I audit.py --output report.json",
    "container_or_os_sandbox_used": false,
    "cpu_seconds": 0.025014,
    "elapsed_seconds": 0.03357779199723154,
    "ended_at_utc": "2026-10-02T21:03:55.367986+00:00",
    "enforced_process_limits": {
      "address_space_bytes": {
        "enforced": false,
        "reason": "current limit exceeds maximum limit"
      },
      "cpu_seconds": 10
    },
    "executable": "/opt/homebrew/opt/python@3.14/bin/python3.14",
    "installed_packages": [],
    "machine": "arm64",
    "manifest_sha256": "7315c20e74fd0043d1434c7936be18748ba41fcf4850bdcfb38c6bc60a8298a9",
    "max_resident_set_size_native_units": 28213248,
    "max_rss_units": "bytes on macOS; kilobytes on Linux",
    "models_called": [],
    "network_used_by_audit": false,
    "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
    "protocol_sha256": "bd7801aff90431df7faff3d33a6d0f5b4d5ae8270e2207873db1b02bee68b0c9",
    "python": "3.14.7 (main, Aug  5 2026, 10:29:49) [Clang 21.0.0 (clang-2100.1.1.101)]",
    "script_sha256": "6eefc22540e3d5eb1cac7bd5f306019ca494c22204e9bae656b8cfb5906f6042",
    "started_at_utc": "2026-10-02T21:03:55.349447+00:00",
    "timezone_for_user_date": "Europe/Zurich",
    "upstream_code_executed": false
  },
  "execution_status": "completed",
  "limitations": [
    "A mismatch compares the pinned available artifact with the cited historical paper and current task documentation. It does not establish the cause, misconduct, incorrect legal labels, or failure of the paper's reported model performance. No model is run.",
    "The pinned artifacts were available at audit time and are not established as the exact dataset revision used for the 2023 experiment.",
    "Table 7 counts are interpreted as test-set slice counts; the paper does not provide a linked checksum of that experiment's split here.",
    "A documentation mismatch does not show that any reported model accuracy is wrong. Saved model predictions were not part of this attempt.",
    "No substantive assessment of legal labels, real-world validity, benchmark contamination or semantic near-duplicates was performed.",
    "The current task documentation has conflicting sample totals, so its statements are compared separately.",
    "Process limits were applied locally; this audit did not test a hosted sandbox, job scheduler or agent reproduction service."
  ],
  "observed": {
    "github_hf_train_bytes_identical": true,
    "test": {
      "duplicate_exact_text_groups": [],
      "duplicate_index_groups": [],
      "duplicate_normalized_text_groups": [],
      "label_counts": {
        "arbitrary": 19,
        "descriptive": 19,
        "fanciful": 19,
        "generic": 19,
        "suggestive": 19
      },
      "rows": 95
    },
    "total_rows": 100,
    "train": {
      "duplicate_exact_text_groups": [],
      "duplicate_index_groups": [],
      "duplicate_normalized_text_groups": [],
      "label_counts": {
        "arbitrary": 1,
        "descriptive": 1,
        "fanciful": 1,
        "generic": 1,
        "suggestive": 1
      },
      "rows": 5
    },
    "train_test_text_overlap": {
      "exact_text": [],
      "normalized_text": []
    }
  },
  "primary_result": "does_not_match_declared_counts",
  "schema_version": "legalarxiv.local-reproduction-report.v1",
  "scientific_scope": "Dataset composition only; no model inference, accuracy reproduction or legal-label validation",
  "study_id": "legalbench-abercrombie-2026-10-02",
  "target": {
    "expected_counts_verified_against_saved_paper": {
      "arbitrary": 18,
      "descriptive": 19,
      "fanciful": 19,
      "generic": 19,
      "suggestive": 20
    },
    "locator": "#S5.T7",
    "paper": "https://arxiv.org/html/2308.11462v1",
    "table_id": "S5.T7.2",
    "tolerance": 0
  }
}