{
  "schema": "repro.report/v3",
  "engine_version": "0.2.1",
  "attempt_id": "arxiv-1812-09970-20261018T010328Z",
  "legacy": true,
  "paper": {
    "arxiv_id": "1812.09970",
    "title": "Synthetic Difference-in-Differences",
    "authors": [
      "Dmitry Arkhangelsky",
      "Susan Carleton Athey",
      "David A. Hirshberg",
      "Guido W. Imbens",
      "Stefan Wager"
    ],
    "year": 2021,
    "doi": "10.1257/aer.20190159",
    "venue": "American Economic Review",
    "openalex_id": "https://openalex.org/W2952814763"
  },
  "code": [
    {
      "commit": "70c1ce3eac58e28c30b67435ca377bb48baa9b8a",
      "full_name": "synth-inference/synthdid",
      "modified_files": 0,
      "path": "/work/synthdid",
      "url": "https://github.com/synth-inference/synthdid",
      "matches_primary": null
    },
    {
      "commit": "6b2706fd7c35f3266048ceb22a7e9a61ae1774da",
      "full_name": "susanathey/MCPanel",
      "modified_files": 0,
      "path": "/work/src/MCPanel",
      "url": "https://github.com/susanathey/MCPanel",
      "matches_primary": null
    },
    {
      "commit": "f4fc95ab49efaad9b6e1728a7c840bc6159501dc",
      "full_name": "cran/glmnet",
      "modified_files": 0,
      "path": "/work/src/glmnet",
      "url": "https://github.com/cran/glmnet",
      "matches_primary": null
    },
    {
      "commit": "2408a924e4dba02c116e935ded7ee1093de24b23",
      "full_name": "synth-inference/synthdid",
      "modified_files": 0,
      "path": "/work/src/synthdid",
      "url": "https://github.com/synth-inference/synthdid",
      "matches_primary": null
    }
  ],
  "code_used": [
    {
      "commit": "2408a924",
      "repo": null,
      "code_version": "primary",
      "source": "method_text",
      "results": [
        45,
        50,
        46,
        47,
        48,
        49,
        51,
        52,
        53,
        54
      ]
    }
  ],
  "primary_code_source": "results",
  "outcome": "reproduced",
  "outcome_label": "Reproduced",
  "qualifiers": [
    "plan_not_sealed",
    "evidence_not_bound",
    "agent_selected_extraction"
  ],
  "qualifier_details": [],
  "evidence_gate_failed": [],
  "assurance": {
    "plan_sealed": false,
    "sealed_before_scored_execution": false,
    "evidence_bound": false,
    "extraction_sealed": false,
    "execution_manifest": false,
    "semantic_rederivation_passed": true,
    "predates": [
      "plan_seal",
      "evidence_binding",
      "execution_manifest",
      "paper_text_hash",
      "scored_job_flag",
      "retry_reasons",
      "extraction_rule"
    ],
    "engine_version": "0.2.1",
    "attempt_engine_version": "0.1.0",
    "setup_outputs_before_seal": null,
    "observations_before_seal": null,
    "observation_provenance": null,
    "plan_hash": null,
    "paper_text_sha256_ok": null
  },
  "plan": null,
  "not_attemptable_reason": null,
  "summary": "Running the authors own Table 1 code (vignettes/paper-results.Rmd) at the synthdid commit named in their session-info, with R 4.0.5 and the pinned MCPanel and glmnet commits, reproduced every entry of Table 1 at the reported precision: SDID -15.6 (8.4), SC -19.6 (9.9), DID -27.3 (17.7), MC -20.2 (11.5), DIFP -11.1 (9.5). Point estimates differ from the paper by at most 0.05, and placebo standard errors by at most 0.05. Deviations were limited to platform (x86_64 Linux instead of an M1 Mac) and one non-estimating build dependency. The paper-era synthdid commit is no longer on any branch of the repository but is still retrievable from GitHub by its hash.",
  "claims": [
    {
      "claim_id": "sdid-estimate",
      "comparison": "within",
      "headline": true,
      "location": "Table 1, p. 8",
      "metric": "SDID point estimate (synthdid_estimate on the California Prop 99 panel)",
      "paper_value": -15.6,
      "quote": "Estimate -15.6 -19.6 -27.3 -20.2 -11.1",
      "statement": "The SDID estimate of the average effect of Proposition 99 on California per-capita cigarette sales over the 12 post-treatment years is -15.6 packs.",
      "tolerance": 0.1,
      "tolerance_rationale": "The estimator is a deterministic optimization on the same public data; the paper rounds to 0.1, so one rounding unit allows for rounding plus minor solver differences.",
      "unit": "packs per capita",
      "registered_event": 9,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.006410256410256411,
      "extraction_registered": false,
      "results": [
        {
          "event": 45,
          "observed_value": -15.603828,
          "delta": -0.0038280000000003866,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 45,
        "observed_value": -15.603828,
        "delta": -0.0038280000000003866,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0002453846153847,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "sdid-se",
      "comparison": "within",
      "headline": true,
      "location": "Table 1, p. 8 (placebo method, Section 5, Algorithm 4)",
      "metric": "sqrt of the placebo variance estimate for SDID",
      "paper_value": 8.4,
      "quote": "Standard error (8.4) (9.9) (17.7) (11.5) (9.5)",
      "statement": "The placebo-method standard error of the SDID estimate is 8.4.",
      "tolerance": 0.84,
      "tolerance_rationale": "Placebo SEs are Monte Carlo estimates from random reassignment; with 200 replications the relative sampling error of an SD estimate is about 5%, so a 10% band (about two sampling SDs) is used.",
      "unit": "packs per capita",
      "registered_event": 10,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.09999999999999999,
      "extraction_registered": false,
      "results": [
        {
          "event": 50,
          "observed_value": 8.367993,
          "delta": -0.03200700000000012,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 50,
        "observed_value": 8.367993,
        "delta": -0.03200700000000012,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 0.9961896428571428,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "sc-estimate",
      "comparison": "within",
      "headline": true,
      "location": "Table 1, p. 8",
      "metric": "SC point estimate (sc_estimate)",
      "paper_value": -19.6,
      "quote": "Estimate -15.6 -19.6 -27.3 -20.2 -11.1",
      "statement": "The synthetic control estimate is -19.6.",
      "tolerance": 0.1,
      "tolerance_rationale": "Deterministic optimization; one rounding unit.",
      "unit": "packs per capita",
      "registered_event": 11,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.00510204081632653,
      "extraction_registered": false,
      "results": [
        {
          "event": 46,
          "observed_value": -19.619663,
          "delta": -0.019662999999997766,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 46,
        "observed_value": -19.619663,
        "delta": -0.019662999999997766,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0010032142857141,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "did-estimate",
      "comparison": "within",
      "headline": true,
      "location": "Table 1, p. 8",
      "metric": "DID point estimate (did_estimate)",
      "paper_value": -27.3,
      "quote": "Estimate -15.6 -19.6 -27.3 -20.2 -11.1",
      "statement": "The difference-in-differences estimate is -27.3.",
      "tolerance": 0.1,
      "tolerance_rationale": "Closed-form estimator on fixed data; one rounding unit.",
      "unit": "packs per capita",
      "registered_event": 12,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.003663003663003663,
      "extraction_registered": false,
      "results": [
        {
          "event": 47,
          "observed_value": -27.349111,
          "delta": -0.049110999999999905,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 47,
        "observed_value": -27.349111,
        "delta": -0.049110999999999905,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0017989377289378,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "mc-estimate",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "MC point estimate (MCPanel nuclear-norm estimator as used in the authors code)",
      "paper_value": -20.2,
      "quote": "Estimate -15.6 -19.6 -27.3 -20.2 -11.1",
      "statement": "The matrix completion estimate is -20.2.",
      "tolerance": 0.5,
      "tolerance_rationale": "MC chooses its regularization by cross-validation over randomly formed folds, so results can move slightly between runs and package versions.",
      "unit": "packs per capita",
      "registered_event": 13,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.024752475247524754,
      "extraction_registered": false,
      "results": [
        {
          "event": 48,
          "observed_value": -20.218135,
          "delta": -0.0181350000000009,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 48,
        "observed_value": -20.218135,
        "delta": -0.0181350000000009,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0008977722772279,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "difp-estimate",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "DIFP point estimate",
      "paper_value": -11.1,
      "quote": "Estimate -15.6 -19.6 -27.3 -20.2 -11.1",
      "statement": "The synthetic control with intercept (DIFP) estimate is -11.1.",
      "tolerance": 0.1,
      "tolerance_rationale": "Deterministic optimization; one rounding unit.",
      "unit": "packs per capita",
      "registered_event": 14,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.009009009009009009,
      "extraction_registered": false,
      "results": [
        {
          "event": 49,
          "observed_value": -11.104645,
          "delta": -0.00464500000000001,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 49,
        "observed_value": -11.104645,
        "delta": -0.00464500000000001,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0004184684684685,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "sc-se",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "sqrt of the placebo variance estimate for SC",
      "paper_value": 9.9,
      "quote": "Standard error (8.4) (9.9) (17.7) (11.5) (9.5)",
      "statement": "The placebo-method standard error of the SC estimate is 9.9.",
      "tolerance": 0.99,
      "tolerance_rationale": "Monte Carlo placebo estimate; 10% band, as for the SDID standard error.",
      "unit": "packs per capita",
      "registered_event": 15,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.09999999999999999,
      "extraction_registered": false,
      "results": [
        {
          "event": 51,
          "observed_value": 9.917426,
          "delta": 0.017426000000000386,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 51,
        "observed_value": 9.917426,
        "delta": 0.017426000000000386,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.001760202020202,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "did-se",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "sqrt of the placebo variance estimate for DID",
      "paper_value": 17.7,
      "quote": "Standard error (8.4) (9.9) (17.7) (11.5) (9.5)",
      "statement": "The placebo-method standard error of the DID estimate is 17.7.",
      "tolerance": 1.77,
      "tolerance_rationale": "Monte Carlo placebo estimate; 10% band, as for the SDID standard error.",
      "unit": "packs per capita",
      "registered_event": 16,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.1,
      "extraction_registered": false,
      "results": [
        {
          "event": 52,
          "observed_value": 17.740267,
          "delta": 0.04026700000000005,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 52,
        "observed_value": 17.740267,
        "delta": 0.04026700000000005,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0022749717514123,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "mc-se",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "sqrt of the placebo variance estimate for MC",
      "paper_value": 11.5,
      "quote": "Standard error (8.4) (9.9) (17.7) (11.5) (9.5)",
      "statement": "The placebo-method standard error of the MC estimate is 11.5.",
      "tolerance": 1.15,
      "tolerance_rationale": "Monte Carlo placebo estimate; 10% band, as for the SDID standard error.",
      "unit": "packs per capita",
      "registered_event": 17,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.09999999999999999,
      "extraction_registered": false,
      "results": [
        {
          "event": 53,
          "observed_value": 11.509864,
          "delta": 0.009864000000000317,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 53,
        "observed_value": 11.509864,
        "delta": 0.009864000000000317,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 1.0008577391304347,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    },
    {
      "claim_id": "difp-se",
      "comparison": "within",
      "headline": false,
      "location": "Table 1, p. 8",
      "metric": "sqrt of the placebo variance estimate for DIFP",
      "paper_value": 9.5,
      "quote": "Standard error (8.4) (9.9) (17.7) (11.5) (9.5)",
      "statement": "The placebo-method standard error of the DIFP estimate is 9.5.",
      "tolerance": 0.95,
      "tolerance_rationale": "Monte Carlo placebo estimate; 10% band, as for the SDID standard error.",
      "unit": "packs per capita",
      "registered_event": 18,
      "quote_verified": true,
      "quote_page": 8,
      "preregistered": true,
      "registered_after_output": false,
      "sealed": false,
      "check_kind": "equivalence",
      "broad_bound": false,
      "tolerance_ratio": 0.09999999999999999,
      "extraction_registered": false,
      "results": [
        {
          "event": 54,
          "observed_value": 9.45566,
          "delta": -0.044340000000000046,
          "consistent": true,
          "evidence_events": [
            42
          ],
          "evidence": null,
          "evidence_binding": "legacy",
          "evidence_verified": true,
          "evidence_reason": "value matched by the legacy rule in events 42",
          "evidence_after_seal": null,
          "evidence_from_unscored_job": false,
          "evidence_scored_execution": null,
          "extraction": "legacy",
          "job_start_event": null,
          "tree_modified_during_job": null,
          "manifest_complete": false,
          "retry_reason": null,
          "rerun_rule_checked": false,
          "rederivation_problems": [],
          "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
          "notes": "",
          "code_version": "primary",
          "code_ref": ""
        }
      ],
      "withdrawn_after_seal": false,
      "final": {
        "event": 54,
        "observed_value": 9.45566,
        "delta": -0.044340000000000046,
        "consistent": true,
        "evidence_events": [
          42
        ],
        "evidence": null,
        "evidence_binding": "legacy",
        "evidence_verified": true,
        "evidence_reason": "value matched by the legacy rule in events 42",
        "evidence_after_seal": null,
        "evidence_from_unscored_job": false,
        "evidence_scored_execution": null,
        "extraction": "legacy",
        "job_start_event": null,
        "tree_modified_during_job": null,
        "manifest_complete": false,
        "retry_reason": null,
        "rerun_rule_checked": false,
        "rederivation_problems": [],
        "method": "Table 1 code extracted verbatim from vignettes/paper-results.Rmd at synthdid commit 2408a924 (R 4.0.5, MCPanel 6b2706fd, glmnet 4.1-1), run once; value printed at full precision by the table1 job. Placebo standard error with set.seed(12345) before each estimator, 200 replications, as in the vignette.",
        "notes": "",
        "code_version": "primary",
        "code_ref": ""
      },
      "earlier_final": null,
      "verdict": "consistent",
      "result_revisions": 0,
      "ratio_to_published": 0.9953326315789474,
      "optional_stopping": false,
      "evidence_problems": [],
      "evidence_ok": true
    }
  ],
  "counts": {
    "claims_scored": 4,
    "consistent": 4,
    "consistent_earlier_code": 0,
    "inconsistent": 0,
    "untested": 0,
    "withdrawn_after_seal": 0,
    "other": {
      "claims": 6,
      "consistent": 6,
      "inconsistent": 0,
      "untested": 0
    }
  },
  "withdrawn": [],
  "protocol": null,
  "deviations": [
    {
      "event": 37,
      "description": "isoband 0.2.5 (from the 2021-08-01 CRAN snapshot) instead of the 0.2.4 in the 2021-05-01 snapshot used for all other CRAN packages.",
      "impact": "none",
      "reason": "isoband 0.2.4 bundles a Catch header that does not compile against glibc 2.34+. isoband is a ggplot2 contouring dependency pulled in by MCPanel and is not used by any estimator."
    },
    {
      "event": 38,
      "description": "Platform differs from the authors Table 1 run: x86_64 Debian 13 with R reference BLAS/LAPACK on a shared-CPU DigitalOcean droplet, versus an M1 MacBook Air with OpenBLAS 0.3.13. R 4.0.5 and the pinned synthdid, MCPanel and glmnet commits match their session-info.",
      "impact": "minor",
      "reason": "Reproduction runs in a Linux sandbox. The authors report cross-platform differences of order 1e-13 except for the MC estimator, whose cross-validation folds use a platform-dependent C++ RNG."
    },
    {
      "event": 39,
      "description": "future.batchtools is not installed or loaded; the vignette imports it alongside rngtools, future and doFuture.",
      "impact": "none",
      "reason": "It is only used to dispatch the Section 3 simulations to a SLURM cluster, which come after Table 1 and are out of scope."
    }
  ],
  "notes": [
    {
      "event": 44,
      "text": "Correction to the deviation at event 38: the sandbox R used OpenBLAS 0.3.29 (pthread build) for BLAS and LAPACK, not the R reference BLAS, per the sessionInfo printed by the table1 job (event 42). The authors used OpenBLAS 0.3.13. The deviation itself (platform differs) stands."
    }
  ],
  "flags": [],
  "compute": {
    "shell_commands": 8,
    "sandbox_seconds": 16.1,
    "jobs": 4,
    "job_seconds": 1512.8,
    "sandbox": {
      "container": "repro-arxiv-1812-09970-20261018t010328z",
      "docker_daemon": {
        "mem_bytes": 8326946816,
        "ncpu": 4
      },
      "effective": {
        "cpus": 3.5,
        "memory_bytes": 6442450944
      },
      "image": "repro-sandbox:0.2",
      "image_id": "sha256:d65217b9b8e6ac7d01c76f7d7770f25c9fd33d0965f08c2d4cefba9bb4c46148",
      "limits": {
        "cpus": 3.5,
        "gpu": null,
        "memory": "6g",
        "network": true
      },
      "target": "cloud",
      "target_detail": {
        "droplet": "repro-sandbox-20261018-0101",
        "image": "ubuntu-24-04-x64",
        "price_hourly_usd": 0.08333,
        "provider": "digitalocean",
        "region": "nyc3",
        "size": "s-4vcpu-8gb-intel"
      },
      "volume": "repro-arxiv-1812-09970-20261018t010328z-work",
      "warnings": []
    },
    "environment_event": 56
  },
  "drivers": [
    "operator:claude-opus-5-5 (Claude Code session, no human in the loop)"
  ],
  "model_usage": {},
  "model_cost_usd": null,
  "started_at": "2026-10-18T01:03:28.762Z",
  "ended_at": "2026-10-18T01:36:21.344Z",
  "log": {
    "events": 59,
    "head_hash": "8b90233f1aba9384bcb8d580e05cbaaf9a5947ebf64d2d62c4f0f4b157f1f7cb"
  },
  "right_of_reply": "Authors may submit a response, which is published alongside this report unedited."
}