{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1803-09015-20261019T161404Z", "legacy": true, "paper": { "arxiv_id": "1803.09015", "title": "Difference-in-Differences with multiple time periods", "authors": [ "Brantly Callaway", "Pedro H. C. Sant'Anna" ], "year": 2020, "doi": "10.1016/j.jeconom.2020.12.001", "venue": "Journal of Econometrics", "openalex_id": "https://openalex.org/W3113269512" }, "code": [ { "commit": "a9a25d4f7672da34df5c57b519f9a9b87ae81c2c", "full_name": "pedrohcgs/CS_RR", "modified_files": 0, "path": "/work/CS_RR", "url": "https://github.com/pedrohcgs/CS_RR", "matches_primary": true }, { "commit": "5e32efd9675578f4662aacfb90695dfd0906c4f0", "full_name": "bcallaway11/did", "modified_files": 0, "path": "/work/did", "url": "https://github.com/bcallaway11/did", "matches_primary": true } ], "code_used": [], "primary_code_source": "protocol", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "claims_registered_after_output", "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "The minimum-wage application (Section 5, Tables 2-3) was rerun with the did package at commit 5e32efd (the last commit before arXiv v4), R 4.0.3 and 2020-12-01 CRAN dependencies, on the 2284-county 2001-2007 panel that co-author Sant'Anna publishes in the CS_RR repository (the bundled mpdta is only a 500-county subset). All 7 pre-registered claims landed within tolerance: the unconditional simple average (-0.051581 vs -0.052), event-study average (-0.089810 vs -0.090), ATT(2004,2007) (-0.135614 vs -0.136), the conditional doubly robust simple average (-0.033117 vs -0.033), both bootstrap SEs, and the Table 2 treated population mean. A post-hoc, unscored comparison found that Table 3's calendar-time rows mix two weightings: Panel (a) matches the paper's eq. (3.8) group weights, while Panel (b) matches the unweighted mean the 2020 package computed.", "claims": [ { "claim_id": "cond-dr-simple-att", "comparison": "within", "headline": true, "location": "Table 3 Panel (b), p. 30", "metric": "aggte(type='simple') overall ATT, DR estimator, covariates region, pop, medinc, pop^2, medinc^2, white, hs, pov", "paper_value": -0.033, "quote": "Simple Weighted Average -0.033", "statement": "Under conditional parallel trends with the doubly robust estimator, the simple weighted average of post-treatment ATT(g,t) is -0.033.", "tolerance": 0.0005, "tolerance_rationale": "Point estimates are deterministic given the data and code (the bootstrap does not affect them); the only allowed gap is the paper's rounding to 3 decimals, so tolerance = 0.0005. The DR first steps (logit propensity score, linear outcome regression) are deterministic.", "unit": "log points", "registered_event": 23, "quote_verified": true, "quote_page": 30, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.015151515151515152, "extraction_registered": false, "results": [ { "event": 44, "observed_value": -0.033117, "delta": -0.00011699999999999905, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~ region + pop + medinc + I(pop^2) + I(medinc^2) + white + hs + pov; aggte(type='simple') overall.att.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 44, "observed_value": -0.033117, "delta": -0.00011699999999999905, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~ region + pop + medinc + I(pop^2) + I(medinc^2) + white + hs + pov; aggte(type='simple') overall.att.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0035454545454545, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "cond-dr-simple-se", "comparison": "within", "headline": false, "location": "Table 3 Panel (b), p. 30", "metric": "bootstrap standard error of aggte(type='simple'), DR conditional PT", "paper_value": 0.007, "quote": "Simple Weighted Average -0.033 (0.007)", "statement": "The clustered multiplier-bootstrap standard error of the conditional DR simple weighted average is 0.007.", "tolerance": 0.00155, "tolerance_rationale": "As for uncond-simple-se: 15% of the paper value (0.00105) for bootstrap randomness plus 0.0005 for 3-decimal rounding.", "unit": "log points", "registered_event": 24, "quote_verified": true, "quote_page": 30, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.22142857142857142, "extraction_registered": false, "results": [ { "event": 46, "observed_value": 0.006506, "delta": -0.0004940000000000005, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, DR with covariates; aggte(type='simple') overall.se (bootstrap).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 46, "observed_value": 0.006506, "delta": -0.0004940000000000005, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, DR with covariates; aggte(type='simple') overall.se (bootstrap).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9294285714285714, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "data-treated-pop-mean", "comparison": "within", "headline": false, "location": "Table 2, p. 28", "metric": "mean of pop across counties with first.treat > 0 (one row per county)", "paper_value": 94.32, "quote": "Population (1000s) 94.32 53.43 40.89 0.00", "statement": "Data identity check: treated counties have mean 2000 population of 94.32 thousand in the main dataset.", "tolerance": 0.005, "tolerance_rationale": "A descriptive mean of fixed data; only the paper's 2-decimal rounding (0.005) is allowed. This checks the released file is the paper's sample.", "unit": "thousands", "registered_event": 25, "quote_verified": true, "quote_page": 28, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 5.301102629346905e-05, "extraction_registered": false, "results": [ { "event": 47, "observed_value": 94.3213, "delta": 0.001300000000000523, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean of pop over the 907 counties with first.treat > 0, one row per county, in /work/run_table3.R.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 47, "observed_value": 94.3213, "delta": 0.001300000000000523, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean of pop over the 907 counties with first.treat > 0, one row per county, in /work/run_table3.R.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0000137828668363, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "uncond-att-2004-2007", "comparison": "within", "headline": true, "location": "Section 5.1, p. 28 (also Table 3 Panel (a) Event Study e=3, p. 30)", "metric": "att_gt ATT(g=2004, t=2007), unconditional PT", "paper_value": -0.136, "quote": "In 2005, teen employment is estimated to be 7.1% lower; in 2006, 12.5% lower; and in 2007, 13.6% lower.", "statement": "Under unconditional parallel trends, the group-time ATT for the 2004 group (Illinois) in 2007 is -0.136.", "tolerance": 0.0005, "tolerance_rationale": "Point estimates are deterministic given the data and code (the bootstrap does not affect them); the only allowed gap is the paper's rounding to 3 decimals, so tolerance = 0.0005.", "unit": "log points", "registered_event": 26, "quote_verified": true, "quote_page": 28, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003676470588235294, "extraction_registered": false, "results": [ { "event": 43, "observed_value": -0.135614, "delta": 0.00038599999999999746, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; att_gt estimate for g=2004, t=2007.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 43, "observed_value": -0.135614, "delta": 0.00038599999999999746, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; att_gt estimate for g=2004, t=2007.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9971617647058824, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "uncond-event-study-overall", "comparison": "within", "headline": true, "location": "Table 3 Panel (a), row 'Event Study', Single Parameters column, p. 30", "metric": "aggte(type='dynamic') overall ATT, unconditional PT", "paper_value": -0.09, "quote": "-0.027 -0.071 -0.125 -0.136 -0.090", "statement": "Under unconditional parallel trends, the event-study single parameter (average of e=0..3 dynamic effects) is -0.090.", "tolerance": 0.0005, "tolerance_rationale": "Point estimates are deterministic given the data and code (the bootstrap does not affect them); the only allowed gap is the paper's rounding to 3 decimals, so tolerance = 0.0005.", "unit": "log points", "registered_event": 27, "quote_verified": true, "quote_page": 30, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.005555555555555556, "extraction_registered": false, "results": [ { "event": 42, "observed_value": -0.08981, "delta": 0.00018999999999999573, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='dynamic', min_e=0) overall.att.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 42, "observed_value": -0.08981, "delta": 0.00018999999999999573, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='dynamic', min_e=0) overall.att.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9978888888888889, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "uncond-simple-att", "comparison": "within", "headline": true, "location": "Section 5.1 p. 28; Table 3 Panel (a) 'Simple Weighted Average', p. 30", "metric": "aggte(type='simple') overall ATT, unconditional PT, never-treated comparison", "paper_value": -0.052, "quote": "The simple average (weighted only by group size) is 5.2% lower teen employment", "statement": "Under unconditional parallel trends with never-treated controls, the simple (group-size weighted) average of post-treatment ATT(g,t) is -0.052 log points of teen employment.", "tolerance": 0.0005, "tolerance_rationale": "Point estimates are deterministic given the data and code (the bootstrap does not affect them); the only allowed gap is the paper's rounding to 3 decimals, so tolerance = 0.0005.", "unit": "log points", "registered_event": 28, "quote_verified": true, "quote_page": 28, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.009615384615384616, "extraction_registered": false, "results": [ { "event": 41, "observed_value": -0.051581, "delta": 0.00041899999999999576, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='simple') overall.att.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 41, "observed_value": -0.051581, "delta": 0.00041899999999999576, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='simple') overall.att.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9919423076923077, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "uncond-simple-se", "comparison": "within", "headline": false, "location": "Table 3 Panel (a), p. 30", "metric": "bootstrap standard error of aggte(type='simple'), unconditional PT", "paper_value": 0.006, "quote": "Simple Weighted Average -0.052 (0.006)", "statement": "The clustered multiplier-bootstrap standard error of the unconditional simple weighted average is 0.006.", "tolerance": 0.0014, "tolerance_rationale": "Bootstrap SEs are random (paper seed unknown); with 1000 multiplier draws the SE estimate itself has Monte Carlo noise of a few percent. Allow 15% of the paper value (0.0009) plus the paper's 3-decimal rounding (0.0005).", "unit": "log points", "registered_event": 29, "quote_verified": true, "quote_page": 30, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.23333333333333334, "extraction_registered": false, "results": [ { "event": 45, "observed_value": 0.006595, "delta": 0.000595, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='simple') overall.se (bootstrap).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 45, "observed_value": 0.006595, "delta": 0.000595, "consistent": true, "evidence_events": [ 35 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 35", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "did @ 5e32efd att_gt (never-treated controls, est_method dr, 1000-draw multiplier bootstrap clustered by county, seed 20201201) on min_wage_CS.rds, run by /work/run_table3.R, xformla ~1; aggte(type='simple') overall.se (bootstrap).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0991666666666666, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 4, "consistent": 4, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 3, "consistent": 3, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": { "earlier_code": "allowed", "primary_code": "github.com/bcallaway11/did at commit 5e32efd9675578f4662aacfb90695dfd0906c4f0 (2020-11-30T22:37:02Z, DESCRIPTION Version 2.0.0), the last commit on or before the tested arXiv version v4 (updated 2020-12-01T16:15:59Z per the arXiv API). Installed from source with R 4.0.3 (Posit build) and CRAN dependencies from the P3M snapshot of 2020-12-01. Data: data/min_wage_CS.rds from github.com/pedrohcgs/CS_RR (commit a9a25d4f), published by co-author Pedro Sant'Anna and described there as the data used in the Callaway and Sant'Anna (2021) application.", "rationale": "The paper's Table 3 was produced by the did package; the rule 'closest release or commit not after the arXiv version' selects 5e32efd, a pre-release commit of did 2.0.0 (CRAN release 2020-12-11, 10 days after v4). The last CRAN release before v4 is did 1.2.3 (2020-02-18, tag v1.2.3, commit 0cc3a89); it may be tried as a declared fallback only for claims whose primary result is inconsistent, and would never replace the primary result." }, "deviations": [ { "event": 48, "description": "Application data taken from data/min_wage_CS.rds in github.com/pedrohcgs/CS_RR (co-author repository, file added 2022-01-02), not from a replication package dated with the paper.", "impact": "minor", "reason": "The paper's 2284-county 2001-2007 panel is not shipped with did (mpdta is a 500-county 2003-2007 subset). The CS_RR file is published by co-author Sant'Anna as the data used in the application and matches the paper's sample size, years, groups and the Table 2 population mean (event 47), but it postdates arXiv v4, so exact identity with the file behind Table 3 cannot be proven." }, { "event": 49, "description": "Environment: R 4.0.3 (Posit build) on x86_64 Debian 13 with GCC 14 and R reference BLAS, on a 4-vCPU DigitalOcean droplet; packages compiled from source with build flags only (/env/Makevars.gcc14: -include cstddef -include cstdint for C++, -Wno-error for three C diagnostics). No source file of did or any dependency was changed.", "impact": "none", "reason": "The authors' platform (a 2.8 GHz i5 laptop, per Section 5) and R version are not stated; R 4.0.3 was the current release on 2020-12-01. The build flags are needed because GCC 14 is stricter than 2020 compilers." }, { "event": 50, "description": "testthat 3.0.4 and isoband 0.2.5 came from the P3M snapshot of 2021-08-01; every other dependency came from the 2020-12-01 snapshot.", "impact": "none", "reason": "testthat 3.0.0 in the 2020-12-01 snapshot vendors a catch.h that uses SIGSTKSZ as a compile-time constant, which glibc >= 2.34 no longer provides (event 18); isoband links to it. Neither package enters the estimation path (isoband is a ggplot2 contour dependency)." }, { "event": 51, "description": "Bootstrap standard errors use our own seed (20201201) with the package default 1000 multiplier draws; the paper's seed is not reported.", "impact": "minor", "reason": "Bootstrap SEs are random draws; they are scored with a loose pre-registered tolerance while point estimates are scored at rounding precision." }, { "event": 52, "description": "The TWFE row of Table 3 and the Supplementary Appendix results (not-yet-treated controls, anticipation) were not attempted.", "impact": "none", "reason": "TWFE is not produced by the did package, and the commit 5e32efd predates did's anticipation argument; the main-text never-treated, no-anticipation specification is the one tested." } ], "notes": [ { "event": 22, "text": "Data decision. The paper application (Section 5, Table 2 note) uses 2284 counties observed 2001-2007. The did package bundled mpdta is a 500-county subset for 2003-2007 and cannot reproduce Table 3. The co-author Pedro Sant Anna publishes data/min_wage_CS.rds in github.com/pedrohcgs/CS_RR (added 2022-01-02 in commit 9d66424, file sha256 ba4497c4...f61a7), introduced there as the data used in Callaway and Sant Anna (2021) application. Event 21 shows it has 2284 counties x 7 years (2001-2007), first.treat groups 0/2004/2006/2007, and the covariates named in the paper (region, pop, medinc, white, hs, pov). It is therefore author-distributed data that can test Table 3; claims are registered against it. Caveat recorded as a deviation: it was released 13 months after arXiv v4 in a separate repository, so it is not provably the exact file behind Table 3; a Table 2 summary statistic is registered as a data-identity check. The simulation tables were not used because no simulation code is distributed by the authors in did or the other author repositories listed in events 9-10." }, { "event": 53, "text": "Unregistered context (event 35): beyond the 7 registered claims, every other Table 3 point estimate produced by the did package was compared to 3 decimals. Panel (a) group effects (-0.091, -0.047, -0.028, overall -0.039), event study e=0..3, and balanced event study (-0.027, -0.071, -0.049) all match; all of Panel (b) matches, including group, event-study, calendar and balanced rows, and the group-time range in the text (0.9% to 7.1% lower). The bootstrap SE of the unconditional simple average, 0.006595, rounds to 0.007 against the paper's 0.006, within the pre-registered SE tolerance. The did package at 5e32efd has no anticipation argument; the main-text specification uses none." }, { "event": 54, "text": "Post-hoc finding (unregistered, events 35-40): Table 3's calendar-time rows use two different weightings. Panel (a) calendar effects (-0.055 in 2006, -0.050 in 2007, overall -0.052) match eq. (3.8)'s P(G=g|G<=t) group weights (our -0.054873, -0.050267, overall -0.052396). Panel (b) calendar effects (-0.030, -0.049, overall -0.033) match an unweighted mean across groups (our -0.029554, -0.049269, overall -0.033440), which is what did at 5e32efd and the CRAN 2.0.0 release compute (R/compute.aggte.R: mean(attt)). Under eq. (3.8) weights Panel (b) would read -0.022 (2006), -0.038 (2007), overall -0.029. The package switched to pg-weighted calendar effects in v2.1 (2021-12). Running did 5e32efd on Panel (a) gives -0.074, -0.078, overall -0.064 for the calendar row. Not scored; reported for the authors' right of reply." } ], "flags": [], "compute": { "shell_commands": 15, "sandbox_seconds": 67.5, "jobs": 2, "job_seconds": 1071.4, "sandbox": { "container": "repro-arxiv-1803-09015-20261019t161404z", "docker_daemon": { "mem_bytes": 8326942720, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 6442450944 }, "image": "repro-sandbox:0.2", "image_id": "sha256:23ad4ea1cf8d1f1eaf81aed5b77d26bbad3b891fcc143edea4d6f08aee4e41c2", "limits": { "cpus": 3.5, "gpu": null, "memory": "6g", "network": true, "shm": "2g" }, "target": "cloud", "target_detail": { "droplet": "repro-sandbox-20261019-1614", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.08333, "provider": "digitalocean", "region": "nyc3", "size": "s-4vcpu-8gb-intel" }, "volume": "repro-arxiv-1803-09015-20261019t161404z-work", "warnings": [] }, "environment_event": 56 }, "drivers": [ "operator:claude-opus-5-5 (Claude Code subagent, AI-driven)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-19T16:14:04.610Z", "ended_at": "2026-10-19T16:43:04.824Z", "log": { "events": 59, "head_hash": "48f311848a704e6fdadc0137322dca91d96cfc105c4e13706dc6e39dbaa4c0d7" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }