{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1905-03222-20261019T233532Z", "legacy": true, "paper": { "arxiv_id": "1905.03222", "title": "Conformalized Quantile Regression", "authors": [ "Yaniv Romano", "Evan Patterson", "Emmanuel J. Candès" ], "year": 2019, "doi": null, "venue": null, "openalex_id": "https://openalex.org/W2946063682" }, "code": [ { "commit": "2b9b5c7d62de7f63af49e86fc570db5c2c77d637", "full_name": "yromano/cqr", "modified_files": 1, "path": "/work/cqr", "url": "https://github.com/yromano/cqr", "matches_primary": true } ], "code_used": [], "primary_code_source": "protocol", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "Clean re-run of attempt arxiv-1905-03222-20261019T184203Z with the same protocol and the same six pre-registered claims, and no foreign or withdrawn claims. Ran the authors' reproducible_experiments/run_experiment.py (commit 2b9b5c7, the last before arXiv v1) unchanged except for the same one-line sklearn Imputer patch, on community, concrete and bike with the same rebuilt Python 3.8 stack, 5 splits (2 for bike). All six claims landed within tolerance: CQR quantile-forest length 1.5070 vs 1.55 (community) and 0.5584 vs 0.56 (concrete), CQR quantile-net 1.8561 vs 1.76 (community) and 0.5704 vs 0.57 (bike), split-conformal RF 1.8578 vs 1.94 (community, longer than CQR as reported), and minimum CQR coverage 89.65% against the 90% nominal level. Compared with the earlier attempt, five values are identical to 4 decimals; only the bike quantile-net cell moved (0.5704 vs 0.5695, coverage 89.65 vs 89.26), likely from a different torch thread count. The paper's Table 1 averages over all 11 datasets were not tested.", "claims": [ { "claim_id": "cqr-coverage-min", "comparison": "at_least", "headline": true, "location": "Section 6.2, p. 10", "metric": "minimum over tested CQR (method, dataset) cells of mean test coverage (%) over random splits", "paper_value": 90, "quote": "all methods based on conformal prediction successfully construct prediction bands at the nominal coverage rate of 90%", "statement": "Conformal methods, including both CQR variants, attain the nominal 90% coverage; observed value is the LOWEST split-averaged test coverage among all CQR (method, dataset) cells run in this attempt.", "tolerance": 2.0, "tolerance_rationale": "Coverage >= 1-alpha is a finite-sample guarantee of the method, so the right test is one-sided (at_least). Split-averaged empirical coverage still fluctuates: on concrete (206 test points, ~412 calibration points) the per-split coverage SD is ~2.6 points, so a 10-20 split mean has SD ~0.6-0.8 points, and taking the minimum over up to 5 cells pushes it lower. A floor of 88% (2 points below nominal) is ~2.5-3 SD and would still catch a real failure of the guarantee such as the 88.51% undercoverage the paper reports for non-conformalized quantile nets.", "unit": "%", "registered_event": 7, "quote_verified": true, "quote_page": 10, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "bound", "broad_bound": false, "tolerance_ratio": 0.022222222222222223, "extraction_registered": false, "results": [ { "event": 44, "observed_value": 89.65, "delta": -0.3499999999999943, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Minimum over the four CQR (method, dataset) cells run (CQR RF community 90.53, CQR RF concrete 90.19, CQR Net community 89.87, CQR Net bike 89.65) of the split-averaged test coverage in percent. Split-conformal RF on community: 89.97; RF-local 90.13. Lowest single split for any CQR cell: 86.89 (CQR RF, concrete, split 4). The earlier attempt's minimum was 89.26 (CQR Net bike).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 44, "observed_value": 89.65, "delta": -0.3499999999999943, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Minimum over the four CQR (method, dataset) cells run (CQR RF community 90.53, CQR RF concrete 90.19, CQR Net community 89.87, CQR Net bike 89.65) of the split-averaged test coverage in percent. Split-conformal RF on community: 89.97; RF-local 90.13. Lowest single split for any CQR cell: 86.89 (CQR RF, concrete, split 4). The earlier attempt's minimum was 89.26 (CQR Net bike).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9961111111111112, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "cqr-nn-bike-length", "comparison": "within", "headline": true, "location": "Figure 4 (bike panel), p. 15", "metric": "average interval length on test set, mean over random splits, response rescaled by mean |y| of training set", "paper_value": 0.57, "quote": "0.57 0.62 0.71 0.76 0.72 0.70 2.26 2.20 Avg. Length Avg. Coverage 0.5 1.0 1.5 2.0 80% 85% 90% 95% 100% CQR Neural Net CQR Random Forests Neural Net Local Neural Net Random Forests Local Random Forests Ridge Local Ridge bike", "statement": "CQR with a quantile neural network gives average 90% interval length 0.57 on the bike sharing dataset (20 random 80/20 splits).", "tolerance": 0.057, "tolerance_rationale": "The paper reports only the 20-split mean (rounded to 2 decimals) and shows split-to-split spread only as boxplots, so no numeric std is available. Tolerance is +/-10% of the paper value: it covers the standard error of a 10-20 split mean on these small datasets (split-level length std is typically 5-10% of the mean, so SE ~1-3%), unseeded/version-dependent NN and forest randomness from a 2026 rebuild of a 2019 stack, and 2-decimal rounding, while staying well below the 15-25% gaps between CQR and split conformal that the paper's comparison rests on. Figures 3-6 print 8 average lengths per dataset; the extracted text lists them in the fixed order CQR Neural Net, CQR Random Forests, Neural Net, Neural Net Local, Random Forests, Random Forests Local, Ridge, Ridge Local. This mapping was fixed BEFORE any run by averaging each position over the 11 datasets: the means (1.397, 1.407, 2.165, 1.805, 2.241, 1.823, 3.062, 2.944) reproduce Table 1 (1.40, 1.41, 2.16, 1.81, 2.24, 1.82, 3.06, 2.94); swapping any pair breaks the match.", "unit": "rescaled response units", "registered_event": 8, "quote_verified": true, "quote_page": 15, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.1, "extraction_registered": false, "results": [ { "event": 43, "observed_value": 0.5704, "delta": 0.00040000000000006697, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-1 of the 'CQR Net' average test-interval length on bike from run_experiment(..., 'cqr_quantile_net', seed) (jobs nn-bike-s0 event 31 and nn-bike-s1 event 33, one split each, ~13 min per split), averaged by summarize.py. Per-split 0.562, 0.579; 2 of 20 splits. The earlier attempt got 0.5695 (0.580, 0.559) on the same splits; the difference comes from the unseeded/thread-dependent network training (this run used 1 torch thread per job, the earlier one 2).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 43, "observed_value": 0.5704, "delta": 0.00040000000000006697, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-1 of the 'CQR Net' average test-interval length on bike from run_experiment(..., 'cqr_quantile_net', seed) (jobs nn-bike-s0 event 31 and nn-bike-s1 event 33, one split each, ~13 min per split), averaged by summarize.py. Per-split 0.562, 0.579; 2 of 20 splits. The earlier attempt got 0.5695 (0.580, 0.559) on the same splits; the difference comes from the unseeded/thread-dependent network training (this run used 1 torch thread per job, the earlier one 2).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.000701754385965, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "cqr-nn-community-length", "comparison": "within", "headline": true, "location": "Figure 5 (community panel), p. 16", "metric": "average interval length on test set, mean over random splits, response rescaled by mean |y| of training set", "paper_value": 1.76, "quote": "1.76 1.55 2.06 1.98 1.94 1.83 1.90 1.85 Avg. Length Avg. Coverage 1.0 1.5 2.0 2.5 80% 85% 90% 95% 100% CQR Neural Net CQR Random Forests Neural Net Local Neural Net Random Forests Local Random Forests Ridge Local Ridge community", "statement": "CQR with a quantile neural network gives average 90% interval length 1.76 on the community dataset (20 random 80/20 splits).", "tolerance": 0.176, "tolerance_rationale": "The paper reports only the 20-split mean (rounded to 2 decimals) and shows split-to-split spread only as boxplots, so no numeric std is available. Tolerance is +/-10% of the paper value: it covers the standard error of a 10-20 split mean on these small datasets (split-level length std is typically 5-10% of the mean, so SE ~1-3%), unseeded/version-dependent NN and forest randomness from a 2026 rebuild of a 2019 stack, and 2-decimal rounding, while staying well below the 15-25% gaps between CQR and split conformal that the paper's comparison rests on. Figures 3-6 print 8 average lengths per dataset; the extracted text lists them in the fixed order CQR Neural Net, CQR Random Forests, Neural Net, Neural Net Local, Random Forests, Random Forests Local, Ridge, Ridge Local. This mapping was fixed BEFORE any run by averaging each position over the 11 datasets: the means (1.397, 1.407, 2.165, 1.805, 2.241, 1.823, 3.062, 2.944) reproduce Table 1 (1.40, 1.41, 2.16, 1.81, 2.24, 1.82, 3.06, 2.94); swapping any pair breaks the match.", "unit": "rescaled response units", "registered_event": 9, "quote_verified": true, "quote_page": 16, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.09999999999999999, "extraction_registered": false, "results": [ { "event": 40, "observed_value": 1.8561, "delta": 0.09610000000000007, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR Net' average test-interval length on community from the authors' run_experiment(..., 'cqr_quantile_net', seed) (job nn-comm-s0-5, event 27), averaged by summarize.py. Per-split lengths 1.829, 1.629, 2.301, 1.666, 1.856 (split SD 0.27); 5 of 20 splits. Same 4-decimal mean as the earlier attempt (1.8561).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 40, "observed_value": 1.8561, "delta": 0.09610000000000007, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR Net' average test-interval length on community from the authors' run_experiment(..., 'cqr_quantile_net', seed) (job nn-comm-s0-5, event 27), averaged by summarize.py. Per-split lengths 1.829, 1.629, 2.301, 1.666, 1.856 (split SD 0.27); 5 of 20 splits. Same 4-decimal mean as the earlier attempt (1.8561).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0546022727272728, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "cqr-rf-community-length", "comparison": "within", "headline": true, "location": "Figure 5 (community panel), p. 16", "metric": "average interval length on test set, mean over random splits, response rescaled by mean |y| of training set", "paper_value": 1.55, "quote": "1.76 1.55 2.06 1.98 1.94 1.83 1.90 1.85 Avg. Length Avg. Coverage 1.0 1.5 2.0 2.5 80% 85% 90% 95% 100% CQR Neural Net CQR Random Forests Neural Net Local Neural Net Random Forests Local Random Forests Ridge Local Ridge community", "statement": "CQR with quantile random forests gives average 90% interval length 1.55 on the community dataset (20 random 80/20 splits).", "tolerance": 0.155, "tolerance_rationale": "The paper reports only the 20-split mean (rounded to 2 decimals) and shows split-to-split spread only as boxplots, so no numeric std is available. Tolerance is +/-10% of the paper value: it covers the standard error of a 10-20 split mean on these small datasets (split-level length std is typically 5-10% of the mean, so SE ~1-3%), unseeded/version-dependent NN and forest randomness from a 2026 rebuild of a 2019 stack, and 2-decimal rounding, while staying well below the 15-25% gaps between CQR and split conformal that the paper's comparison rests on. Figures 3-6 print 8 average lengths per dataset; the extracted text lists them in the fixed order CQR Neural Net, CQR Random Forests, Neural Net, Neural Net Local, Random Forests, Random Forests Local, Ridge, Ridge Local. This mapping was fixed BEFORE any run by averaging each position over the 11 datasets: the means (1.397, 1.407, 2.165, 1.805, 2.241, 1.823, 3.062, 2.944) reproduce Table 1 (1.40, 1.41, 2.16, 1.81, 2.24, 1.82, 3.06, 2.94); swapping any pair breaks the match.", "unit": "rescaled response units", "registered_event": 10, "quote_verified": true, "quote_page": 16, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.09999999999999999, "extraction_registered": false, "results": [ { "event": 39, "observed_value": 1.507, "delta": -0.04300000000000015, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR RF' average test-interval length on community, from the authors' run_experiment(dataset='community', test_method='cqr_quantile_forest', seed) rows in results.csv (job event 36), averaged by summarize.py. Per-split lengths 1.571, 1.344, 1.536, 1.517, 1.566; 5 of 20 splits. Same value as the earlier attempt (1.507).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 39, "observed_value": 1.507, "delta": -0.04300000000000015, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR RF' average test-interval length on community, from the authors' run_experiment(dataset='community', test_method='cqr_quantile_forest', seed) rows in results.csv (job event 36), averaged by summarize.py. Per-split lengths 1.571, 1.344, 1.536, 1.517, 1.566; 5 of 20 splits. Same value as the earlier attempt (1.507).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.972258064516129, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "cqr-rf-concrete-length", "comparison": "within", "headline": true, "location": "Figure 5 (concrete panel), p. 16", "metric": "average interval length on test set, mean over random splits, response rescaled by mean |y| of training set", "paper_value": 0.56, "quote": "0.48 0.56 0.50 0.49 0.54 0.53 0.99 0.92 Avg. Length Avg. Coverage 0.4 0.6 0.8 1.0 80% 85% 90% 95% 100% CQR Neural Net CQR Random Forests Neural Net Local Neural Net Random Forests Local Random Forests Ridge Local Ridge concrete", "statement": "CQR with quantile random forests gives average 90% interval length 0.56 on the concrete dataset (20 random 80/20 splits).", "tolerance": 0.056, "tolerance_rationale": "The paper reports only the 20-split mean (rounded to 2 decimals) and shows split-to-split spread only as boxplots, so no numeric std is available. Tolerance is +/-10% of the paper value: it covers the standard error of a 10-20 split mean on these small datasets (split-level length std is typically 5-10% of the mean, so SE ~1-3%), unseeded/version-dependent NN and forest randomness from a 2026 rebuild of a 2019 stack, and 2-decimal rounding, while staying well below the 15-25% gaps between CQR and split conformal that the paper's comparison rests on. Figures 3-6 print 8 average lengths per dataset; the extracted text lists them in the fixed order CQR Neural Net, CQR Random Forests, Neural Net, Neural Net Local, Random Forests, Random Forests Local, Ridge, Ridge Local. This mapping was fixed BEFORE any run by averaging each position over the 11 datasets: the means (1.397, 1.407, 2.165, 1.805, 2.241, 1.823, 3.062, 2.944) reproduce Table 1 (1.40, 1.41, 2.16, 1.81, 2.24, 1.82, 3.06, 2.94); swapping any pair breaks the match.", "unit": "rescaled response units", "registered_event": 11, "quote_verified": true, "quote_page": 16, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.09999999999999999, "extraction_registered": false, "results": [ { "event": 42, "observed_value": 0.5584, "delta": -0.0016000000000000458, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR RF' average test-interval length on concrete from run_experiment(..., 'cqr_quantile_forest', seed) (job event 36), averaged by summarize.py. 5 of 20 splits. Same as the earlier attempt (0.5584).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 42, "observed_value": 0.5584, "delta": -0.0016000000000000458, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'CQR RF' average test-interval length on concrete from run_experiment(..., 'cqr_quantile_forest', seed) (job event 36), averaged by summarize.py. 5 of 20 splits. Same as the earlier attempt (0.5584).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9971428571428571, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "split-rf-community-length", "comparison": "within", "headline": false, "location": "Figure 5 (community panel), p. 16", "metric": "average interval length on test set, mean over random splits, response rescaled by mean |y| of training set", "paper_value": 1.94, "quote": "1.76 1.55 2.06 1.98 1.94 1.83 1.90 1.85 Avg. Length Avg. Coverage 1.0 1.5 2.0 2.5 80% 85% 90% 95% 100% CQR Neural Net CQR Random Forests Neural Net Local Neural Net Random Forests Local Random Forests Ridge Local Ridge community", "statement": "Standard split conformal prediction with a conditional-mean random forest gives average 90% interval length 1.94 on the community dataset, longer than CQR random forests (1.55).", "tolerance": 0.194, "tolerance_rationale": "The paper reports only the 20-split mean (rounded to 2 decimals) and shows split-to-split spread only as boxplots, so no numeric std is available. Tolerance is +/-10% of the paper value: it covers the standard error of a 10-20 split mean on these small datasets (split-level length std is typically 5-10% of the mean, so SE ~1-3%), unseeded/version-dependent NN and forest randomness from a 2026 rebuild of a 2019 stack, and 2-decimal rounding, while staying well below the 15-25% gaps between CQR and split conformal that the paper's comparison rests on. Figures 3-6 print 8 average lengths per dataset; the extracted text lists them in the fixed order CQR Neural Net, CQR Random Forests, Neural Net, Neural Net Local, Random Forests, Random Forests Local, Ridge, Ridge Local. This mapping was fixed BEFORE any run by averaging each position over the 11 datasets: the means (1.397, 1.407, 2.165, 1.805, 2.241, 1.823, 3.062, 2.944) reproduce Table 1 (1.40, 1.41, 2.16, 1.81, 2.24, 1.82, 3.06, 2.94); swapping any pair breaks the match.", "unit": "rescaled response units", "registered_event": 12, "quote_verified": true, "quote_page": 16, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.1, "extraction_registered": false, "results": [ { "event": 41, "observed_value": 1.8578, "delta": -0.08220000000000005, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'RF' (standard split conformal, conditional-mean random forest) average test-interval length on community, from run_experiment(..., 'random_forest', seed) (job event 36), averaged by summarize.py. Per-split 1.795, 1.859, 1.834, 1.885, 1.915; CQR RF on the same splits is 1.5070, i.e. shorter, as the paper reports. RF-local (not a registered claim) gave 1.7613 vs paper 1.83. Same as the earlier attempt (1.8578).", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 41, "observed_value": 1.8578, "delta": -0.08220000000000005, "consistent": true, "evidence_events": [ 38 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 38", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over splits 0-4 of the 'RF' (standard split conformal, conditional-mean random forest) average test-interval length on community, from run_experiment(..., 'random_forest', seed) (job event 36), averaged by summarize.py. Per-split 1.795, 1.859, 1.834, 1.885, 1.915; CQR RF on the same splits is 1.5070, i.e. shorter, as the paper reports. RF-local (not a registered claim) gave 1.7613 vs paper 1.83. Same as the earlier attempt (1.8578).", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9576288659793815, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 5, "consistent": 5, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 1, "consistent": 1, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": { "earlier_code": "not_allowed", "primary_code": "github.com/yromano/cqr at 2b9b5c7d62de7f63af49e86fc570db5c2c77d637 (2019-05-03), the last commit on or before the arXiv v1 date 2019-05-08 (v1 is the only arXiv version). Experiments run through the authors' reproducible_experiments/run_experiment.py with its hard-coded settings.", "rationale": "arXiv 1905.03222 has only v1 (8 May 2019); 2b9b5c7 is the last commit before it and already contains reproducible_experiments/ with the paper's hyper-parameters. Later commits (2019-06 bug fixes, 2022 MEPS cleaning fix, 2026 numpy fix) postdate the paper and are not used." }, "deviations": [ { "event": 23, "description": "One-line patch to the authors' datasets/datasets.py (community loader): `from sklearn.preprocessing import Imputer` / `Imputer(missing_values='NaN', strategy='mean', axis=0)` replaced by `from sklearn.impute import SimpleImputer as Imputer` / `Imputer(missing_values=np.nan, strategy='mean')`. The full git diff is in the output of event 16; it is byte-identical to the patch in the earlier attempt arxiv-1905-03222-20261019T184203Z.", "impact": "none", "reason": "sklearn.preprocessing.Imputer was removed in scikit-learn 0.22, the oldest release with Python 3.8 wheels available in the sandbox. SimpleImputer performs the same column-mean imputation of the single column with missing values (OtherPerCap)." }, { "event": 24, "description": "Software environment rebuilt in 2026 (same stack as the earlier attempt): Python 3.8.20, numpy 1.18.5, scipy 1.4.1, scikit-learn 0.22.2.post1, pandas 1.0.5, torch 1.4.0+cpu, and scikit-garden built from its git master (fbd9a5b, 2022) because the 0.1.3 PyPI sdist ships pre-generated C that does not compile on Python >= 3.7. The quantile-forest Python code in master differs from 0.1.3 only in import paths and removal of the `presort` argument (diff in event 15). The paper does not state library versions; the 2019 originals were likely Python 3.6/3.7, scikit-learn 0.20/0.21 and torch 1.0/1.1. GCC needed `-Wno-error=incompatible-pointer-types -Wno-error=int-conversion` to compile the Cython output (build flags only).", "impact": "minor", "reason": "The exact 2019 environment is not recorded and Python 3.7 is not available from the sandbox's package manager; this is the closest buildable stack, reused unchanged from the earlier attempt so the two runs are comparable." }, { "event": 25, "description": "Only 3 of the paper's 11 datasets (community, concrete, bike) and 3 of its 8 interval methods (CQR quantile random forest, CQR quantile neural net, split-conformal random forest; the RF-local variant runs alongside split RF because the authors' script fits both) are run. The paper's Table 1 numbers are averages over all 11 datasets and are not tested here; per-dataset claims are taken from the numbers printed in Figures 4-5.", "impact": "minor", "reason": "Same scope as the earlier attempt, within a budget of under 4 hours on one 4 vCPU box (3.5 CPUs to the sandbox). MEPS 19/20/21 require separately downloading and processing AHRQ survey files (not bundled), blog and facebook_1/2 are separate UCI downloads, and the larger datasets with 1000-tree quantile forests and up-to-1000-epoch networks exceed the CPU budget." }, { "event": 26, "description": "Fewer random training/test splits than the paper's 20: splits (seeds) 0-4 for split-conformal RF on community, CQR quantile RF on community and concrete, and CQR quantile net on community; only splits 0-1 for CQR quantile net on bike. The authors' run_experiment() is called unchanged per seed, so each seed reproduces the paper's split for that seed. The split counts are the same as the earlier attempt and were fixed before any job started (events 19-22).", "impact": "minor", "reason": "CPU budget: in the earlier attempt one bike split of the CQR quantile net (1000 full epochs plus a refit, as the authors' learner does) took about 40 minutes on CPU, and the forest cells about 56 minutes for 5 splits." } ], "notes": [ { "event": 5, "text": "Clean re-run of attempt arxiv-1905-03222-20261019T184203Z, whose log contains 7 withdrawn placeholder claims from an operator error; same protocol and claims, registered fresh." }, { "event": 47, "text": "Observation on the authors code (not changed), confirmed in event 46: run_experiment.py fits the feature StandardScaler and the response scale (mean |y|) on the full 80% training split, i.e. proper-training plus calibration, whereas footnote 5 of the paper says these statistics are computed only on the proper training set. This attempt keeps the authors code as is, as the earlier attempt did." }, { "event": 48, "text": "Comparison with the earlier attempt arxiv-1905-03222-20261019T184203Z: same commit, environment, patch (byte-identical diff), driver and summary scripts (byte-identical), datasets, splits and claims. The forest cells and the community quantile-net cell gave the same 4-decimal means as before (1.5070, 0.5584, 1.8578, 1.8561), so those runs are deterministic per seed on this stack. Only the bike quantile-net cell moved (0.5704 vs 0.5695 length, 89.65 vs 89.26 coverage): this run trained the two bike splits as separate one-thread jobs in parallel on a dedicated box, while the earlier run used two torch threads on a shared box, and the authors do not seed the network. Execution layout only; no setting of the paper or the authors code differs." }, { "event": 50, "text": "Correction to the note at event 48: the authors DO seed the network (run_experiment.py lines 55-56 call np.random.seed(seed) and torch.manual_seed(seed), event 49). The bike difference is therefore not from missing seeds; the likely cause is floating-point non-associativity from a different torch/BLAS thread count (1 thread per job here vs 2 in the earlier run), which over 1000 epochs on bike can change the training path. The community quantile-net per-split lengths also differ from the earlier run, but only around the 7th significant digit (e.g. 1.8287340 vs 1.8287343 for split 0), consistent with this explanation. This cause is inferred, not tested." } ], "flags": [], "compute": { "shell_commands": 8, "sandbox_seconds": 74.5, "jobs": 4, "job_seconds": 3851.7, "sandbox": { "container": "repro-arxiv-1905-03222-20261019t233532z", "docker_daemon": { "mem_bytes": 8326938624, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 6442450944 }, "image": "repro-sandbox:0.2", "image_id": "sha256:319dfea6bd8dfdd0aca73f261f925aa0af8c7a1c705a800e6566459b53ae4e1b", "limits": { "cpus": 3.5, "gpu": null, "memory": "6g", "network": true, "shm": "2g" }, "target": "cloud", "target_detail": { "box": "arxiv-1905-03222-20261019t233532z", "droplet": "repro-arxiv-1905-03222-20261019-2335", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.08333, "provider": "digitalocean", "region": "nyc3", "size": "s-4vcpu-8gb-intel" }, "volume": "repro-arxiv-1905-03222-20261019t233532z-work", "warnings": [] }, "environment_event": 52 }, "drivers": [ "operator:claude-opus-5-5 (Claude Code subagent, AI-driven)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-19T23:35:32.294Z", "ended_at": "2026-10-20T00:16:09.797Z", "log": { "events": 55, "head_hash": "1a032d5f5c4c462b52fd0bd3b2c0405dabe9029151ad90ac318c6df4d3518877" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }