{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1910-13051-20261019T203424Z", "legacy": true, "paper": { "arxiv_id": "1910.13051", "title": "ROCKET: exceptionally fast and accurate time series classification using random convolutional kernels", "authors": [ "Angus Dempster", "François Petitjean", "Geoffrey I. Webb" ], "year": 2020, "doi": "10.1007/s10618-020-00701-z", "venue": "Data Mining and Knowledge Discovery", "openalex_id": "https://openalex.org/W2982438846" }, "code": [ { "commit": "4c2461070e59c10d467b94a5304ca2df70925969", "full_name": "angus924/rocket", "modified_files": 0, "path": "/work/rocket", "url": "https://github.com/angus924/rocket", "matches_primary": true } ], "code_used": [], "primary_code_source": "protocol", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "claims_registered_after_output", "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "Corrective re-run that supersedes arxiv-1910-13051-20261017T030206Z. The primary code version was declared first: angus924/rocket 4c24610, the first code commit, 21 hours after arXiv v1; the earlier attempt ran 2022 HEAD with the July 2020 rewrite. All claims were then registered, including the best-mean-rank tie rule taken from the paper before any result. Running the authors' 2019 bake-off code on all 85 UCR datasets (10 runs, single-core processes, dedicated box), mean accuracy is 0.8517 against 0.8510 in Table 1, with a mean absolute per-dataset difference of 0.20 points. Using the paper's fractional-rank method, ROCKET has the strictly best mean rank (571/170 vs 579/170 for TS-CHIEF), so the tie rule (a tie would count as not reproduced) was not triggered. The earlier attempt's tie appears to reflect run-to-run variation of unseeded random kernels on a margin the paper calls not significant. Single-core compute times (40.5 min total, 2.35 min ElectricDevices training, 1.81 min HandOutlines) are within the paper's figures on newer hardware.", "claims": [ { "claim_id": "bakeoff-mean-accuracy", "comparison": "within", "headline": true, "location": "Table 1, pp. 25-26 (mean of the Rocket column, computed from Table 1 at event 19; the paper does not print the mean itself)", "metric": "Mean over the 85 bake-off datasets of ROCKET test accuracy (each dataset: mean of 10 runs with fresh random kernels, original train/test split), unrounded", "paper_value": 0.851002, "quote": "Table 1: Classification Accuracy, ‘Bake Off’ Datasets", "statement": "Averaged over the 85 bake-off datasets, ROCKET (10,000 kernels, ridge classifier, 10 runs) attains the per-dataset test accuracies reported in Table 1, whose mean is 0.8510.", "tolerance": 0.005, "tolerance_rationale": "Same tolerance as the earlier attempt, for comparability. Table 1 values are 10-run means with different random kernels; run-to-run spread of roughly 0.5-1 pp per dataset averages down to well under 0.1 pp over 85 datasets, so the 0.5 pp band mainly allows for library and dataset-release differences (UCR 2018 release, current libraries).", "unit": "accuracy", "registered_event": 25, "quote_verified": true, "quote_page": 25, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.005875426849760635, "extraction_registered": false, "results": [ { "event": 47, "observed_value": 0.851669, "delta": 0.0006669999999999732, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over the 85 bake-off datasets of the unrounded 10-run mean test accuracy (exact k/(10 n_test) recovered from the authors CSV), all 85 run with the authors code at 4c24610 (dataset tuple only changed), computed by analysis/calc.py.", "notes": "Per dataset at 4 dp: 38 higher, 15 equal, 32 lower than Table 1. Earlier attempt (HEAD 36bd908): 0.850932.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 47, "observed_value": 0.851669, "delta": 0.0006669999999999732, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over the 85 bake-off datasets of the unrounded 10-run mean test accuracy (exact k/(10 n_test) recovered from the authors CSV), all 85 run with the authors code at 4c24610 (dataset tuple only changed), computed by analysis/calc.py.", "notes": "Per dataset at 4 dp: 38 higher, 15 equal, 32 lower than Table 1. Earlier attempt (HEAD 36bd908): 0.850932.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.000783781941758, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "bakeoff-best-mean-rank", "comparison": "at_most", "headline": true, "location": "Section 4, p. 11 (also p. 11: 'ranks slightly ahead of HIVE-COTE, TS-CHIEF, and InceptionTime'); Figure 1, p. 1; ranking method p. 11 (Demsar 2006) and p. 18 (rank figures produced with code from Ismail Fawaz et al. 2019a)", "metric": "Number of the other seven Table 1 classifiers (BOSS, ST, HIVE-COTE, ResNet, Proximity Forest, TS-CHIEF, InceptionTime) whose mean rank is lower than or equal to ROCKET's. Ranking method, fixed before any result: per dataset, the 8 classifiers are ranked by accuracy (1 = most accurate) with tied classifiers sharing the average of the positions they span (fractional ranks as in Demsar 2006 and the Ismail Fawaz et al. 2019a critical-difference code the paper used, which ranks with pandas rank, method 'average'); mean rank = mean over the 85 datasets, in exact rational arithmetic. The reproduced ROCKET 10-run mean accuracy replaces the Rocket column after rounding half-up to 4 decimals (the precision at which Table 1 reports all eight classifiers); the other seven columns are Table 1 as printed. Computed by analysis/calc.py (event 18).", "paper_value": 0, "quote": "obtaining the best mean rank over the 85 ‘bake off’ datasets", "statement": "ROCKET has the strictly best (lowest) mean rank of the eight classifiers in Table 1 over the 85 bake-off datasets. A tie for the best mean rank does NOT count as reproducing this claim.", "tolerance": 0, "tolerance_rationale": "Tie rule, decided from the paper before any result: the paper's claim is a strict lead. It says ROCKET obtains 'the best mean rank' and, describing Figure 1, that ROCKET 'ranks slightly ahead of' HIVE-COTE, TS-CHIEF and InceptionTime (p. 11). A mean rank equal to another classifier's is not ahead of it, so a shared best mean rank counts as NOT reproduced (metric >= 1). Ties within a dataset are averaged, which is the paper's own ranking convention (Demsar 2006; Ismail Fawaz et al. 2019a code), not a choice made here. Paper-side check from Table 1 under exactly this rule (event 19): ROCKET 571/170 = 3.3588, TS-CHIEF 581/170 = 3.4176, metric 0. Sensitivity, stated in advance: the margin is 10/170 of a mean rank, about 5 dataset-level half-rank swaps against TS-CHIEF, and ROCKET's kernels are unseeded, so ordinary run-to-run variation can flip this binary check; the paper itself says the differences among the top four are not statistically significant.", "unit": "classifiers", "registered_event": 26, "quote_verified": true, "quote_page": 11, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "bound", "broad_bound": false, "tolerance_ratio": null, "extraction_registered": false, "results": [ { "event": 48, "observed_value": 0, "delta": 0.0, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Pre-registered rule (event 26, code at event 18): reproduced Rocket 10-run means rounded half-up to 4 dp substituted into Table 1, fractional per-dataset ranks with ties averaged, exact mean ranks over 85 datasets; count of other classifiers with mean rank <= Rocket's.", "notes": "Rocket 571/170 = 3.3588 (identical to the Table 1 value), TS-CHIEF 579/170 = 3.4059, InceptionTime 619/170, HIVE-COTE 623/170. No tie: Rocket is strictly best, so the tie rule did not come into play. The earlier attempt's reproduced accuracies had tied Rocket and TS-CHIEF at 288/85.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 48, "observed_value": 0, "delta": 0.0, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Pre-registered rule (event 26, code at event 18): reproduced Rocket 10-run means rounded half-up to 4 dp substituted into Table 1, fractional per-dataset ranks with ties averaged, exact mean ranks over 85 datasets; count of other classifiers with mean rank <= Rocket's.", "notes": "Rocket 571/170 = 3.3588 (identical to the Table 1 value), TS-CHIEF 579/170 = 3.4059, InceptionTime 619/170, HIVE-COTE 623/170. No tie: Rocket is strictly best, so the tie rule did not come into play. The earlier attempt's reproduced accuracies had tied Rocket and TS-CHIEF at 288/85.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": null, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "bakeoff-per-dataset-agreement", "comparison": "at_most", "headline": false, "location": "Section 4.1.1, p. 11; Table 1", "metric": "Mean over the 85 datasets of |reproduced 10-run mean accuracy (unrounded) - Table 1 accuracy|", "paper_value": 0, "quote": "The results presented for Rocket are mean results over 10 runs (using a different set of random kernels for each run).", "statement": "Per-dataset ROCKET accuracies in Table 1 are reproducible: the mean absolute difference between reproduced and reported accuracy across the 85 datasets is small.", "tolerance": 0.01, "tolerance_rationale": "Same as the earlier attempt: with 10-run means, sampling noise alone gives per-dataset differences of roughly 0.2-0.5 pp on typical datasets and more on small test sets; up to 1 pp mean absolute difference is treated as agreement.", "unit": "accuracy", "registered_event": 27, "quote_verified": true, "quote_page": 11, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "bound", "broad_bound": true, "tolerance_ratio": null, "extraction_registered": false, "results": [ { "event": 49, "observed_value": 0.001975, "delta": 0.001975, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over 85 datasets of |unrounded reproduced 10-run mean - Table 1|, analysis/calc.py.", "notes": "Max absolute difference 0.0150 (Car). Earlier attempt: 0.002117.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 49, "observed_value": 0.001975, "delta": 0.001975, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Mean over 85 datasets of |unrounded reproduced 10-run mean - Table 1|, analysis/calc.py.", "notes": "Max absolute difference 0.0150 (Car). Earlier attempt: 0.002117.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": null, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "bakeoff-total-compute-minutes", "comparison": "at_most", "headline": true, "location": "Section 1, p. 2", "metric": "Sum over the 85 datasets of the authors' script time_training_seconds + time_test_seconds (per-run means over 10 runs; kernel transform, ridge fit and scoring included; data loading and JIT compilation excluded), in minutes, with every process single-threaded (NUMBA_NUM_THREADS=1, BLAS threads 1) on a dedicated box", "paper_value": 110, "quote": "The total compute time (training and test) for Rocket on all 85 ‘bake off’ datasets is 1 hour 50 minutes", "statement": "Total compute time (training and test, single CPU core per dataset per run, averaged over runs) for ROCKET over all 85 bake-off datasets is 1 hour 50 minutes.", "tolerance": 55, "tolerance_rationale": "Measured fairly here: a dedicated droplet (4 vCPU AMD EPYC 9555P, no other attempt on it), every dataset single-threaded, at most 3 concurrent processes under a 3.5-CPU quota so each has a full core, and enough memory that no workaround is needed. The paper used Xeon E5-2680 v3 / Gold 6150 cores; this CPU is newer and likely faster, so the check is an upper bound (at_most) with the same 50% headroom as the earlier attempt, for comparability.", "unit": "minutes", "registered_event": 28, "quote_verified": true, "quote_page": 2, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "bound", "broad_bound": false, "tolerance_ratio": 0.5, "extraction_registered": false, "results": [ { "event": 50, "observed_value": 40.54, "delta": -69.46000000000001, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Sum over 85 datasets of the authors script time_training_seconds + time_test_seconds (10-run means), single-threaded processes, 3 concurrent on a dedicated 4-vCPU AMD EPYC 9555P box, divided by 60.", "notes": "Hardware is much newer than the paper's 2014-2017 Xeons, so this confirms the upper bound only. Earlier attempt: 111.18 (inflated by memory pressure and a disk-backed workaround).", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 50, "observed_value": 40.54, "delta": -69.46000000000001, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Sum over 85 datasets of the authors script time_training_seconds + time_test_seconds (10-run means), single-threaded processes, 3 concurrent on a dedicated 4-vCPU AMD EPYC 9555P box, divided by 60.", "notes": "Hardware is much newer than the paper's 2014-2017 Xeons, so this confirms the upper bound only. Earlier attempt: 111.18 (inflated by memory pressure and a disk-backed workaround).", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.36854545454545456, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "electricdevices-train-minutes", "comparison": "at_most", "headline": false, "location": "Section 1, p. 2", "metric": "time_training_seconds for ElectricDevices from the authors' script (training transform + ridge fit, mean per run over 10 runs, single-threaded), in minutes", "paper_value": 6, "quote": "6 minutes for the ‘bake off’ dataset with the largest training set", "statement": "Training ROCKET on ElectricDevices (8,926 training examples) takes about 6 minutes on a single core.", "tolerance": 3, "tolerance_rationale": "Same hardware caveat and 50% headroom as the total-time claim (and as the earlier attempt, which could not test this claim).", "unit": "minutes", "registered_event": 29, "quote_verified": true, "quote_page": 2, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "bound", "broad_bound": false, "tolerance_ratio": 0.5, "extraction_registered": false, "results": [ { "event": 51, "observed_value": 2.35, "delta": -3.65, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "time_training_seconds for ElectricDevices (training transform + ridge fit, 10-run mean) from the authors script, single-threaded, in-memory path (peak RSS 7052 MB), divided by 60.", "notes": "Earlier attempt could not test this claim (low-memory workaround only).", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 51, "observed_value": 2.35, "delta": -3.65, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "time_training_seconds for ElectricDevices (training transform + ridge fit, 10-run mean) from the authors script, single-threaded, in-memory path (peak RSS 7052 MB), divided by 60.", "notes": "Earlier attempt could not test this claim (low-memory workaround only).", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.39166666666666666, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "handoutlines-train-minutes", "comparison": "at_most", "headline": false, "location": "Section 1, p. 2", "metric": "time_training_seconds for HandOutlines from the authors' script (training transform + ridge fit, mean per run over 10 runs, single-threaded), in minutes", "paper_value": 4.8667, "quote": "4 minutes and 52 seconds for the ‘bake off’ dataset with the longest time series", "statement": "Training ROCKET on HandOutlines (series length 2,709) takes 4 minutes 52 seconds on a single core.", "tolerance": 2.4333, "tolerance_rationale": "Same hardware caveat and 50% headroom as the total-time claim.", "unit": "minutes", "registered_event": 30, "quote_verified": true, "quote_page": 2, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "bound", "broad_bound": false, "tolerance_ratio": 0.4999897260977665, "extraction_registered": false, "results": [ { "event": 52, "observed_value": 1.81, "delta": -3.0566999999999998, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "time_training_seconds for HandOutlines (10-run mean) from the authors script, single-threaded, divided by 60.", "notes": "Earlier attempt: 4.03.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 52, "observed_value": 1.81, "delta": -3.0566999999999998, "consistent": true, "evidence_events": [ 46 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 46", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "time_training_seconds for HandOutlines (10-run mean) from the authors script, single-threaded, divided by 60.", "notes": "Earlier attempt: 4.03.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.37191526085437776, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 3, "consistent": 3, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 3, "consistent": 3, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": { "earlier_code": "not_allowed", "primary_code": "angus924/rocket at commit 4c2461070e59c10d467b94a5304ca2df70925969 ('Initial commit' of the code, 2019-10-30T11:13:58+11:00 = 2019-10-30T00:13:58Z), running its code/reproduce_experiments_bakeoff.py and code/rocket_functions.py. Rule: the last authors commit on or before the arXiv v1 date (v1 submitted 2019-10-29T02:48:56Z, export.arxiv.org, event 6). Applying that rule literally gives 7b17f0a (2019-10-29T13:29:43Z, 'Create LICENSE'), and the only other earlier commit is 47152a6 (2019-10-21, README only); neither contains any code. The first commit containing code is 4c24610, 21 hours after v1 was submitted, and it is the bake-off code released to accompany v1 (its headers cite arXiv:1910.13051). It is therefore the closest available version to the paper date and is fixed here as primary. Its bake-off code is byte-identical up to 531be13 (2019-11-06), the last commit before the July 2020 journal-version rewrite (git diff --stat empty, event 7).", "rationale": "The claims are tested against arXiv v1 (the PDF fetched for this attempt, identical to the earlier attempt's PDF). There is no earlier code to fall back to: the only commits before 4c24610 contain a README and a LICENSE. Later code is not a fallback either: the earlier attempt (arxiv-1910-13051-20261017T030206Z) ran HEAD 36bd908 (2022-04-11), whose code differs materially from what accompanied v1. At HEAD, reproduce_experiments_bakeoff.py is deleted and replaced by reproduce_experiments_ucr.py (added in 41cfd4b/e2349e0, July 2020), and rocket_functions.py was rewritten (101 lines changed: flat weight storage, dilation computed differently, padding folded into the output-length formula, and the 2019 apply_kernels is parallel=True with prange over examples). So this re-run uses different code from the earlier attempt. Fixing one primary version in advance, with no fallback, avoids choosing among versions after seeing results." }, "deviations": [ { "event": 53, "description": "Library versions differ from the paper era: Python 3.10, numpy 1.23.5, scipy 1.9.3, scikit-learn 1.1.3, pandas 1.5.3, numba 0.56.4 (event 11), identical to the earlier attempt. The repository pins no versions.", "impact": "minor", "reason": "A paper-era Python 3.7 environment could not be installed (no 3.7 download, event 10). scikit-learn 1.1.3 is the newest release that still accepts RidgeClassifierCV(normalize=True), and numpy 1.23.5 still has np.int and np.NINF, so the 2019 code runs unmodified. Keeping the earlier attempt's environment isolates the code-version change." }, { "event": 54, "description": "Data come from the UCR archive's own 2018 release (UCRArchive_2018.zip from cs.ucr.edu, sha256 a7163f6e..., event 14), tab-separated files renamed to the _TRAIN.txt/_TEST.txt names the 2019 script loads, instead of the txt files from timeseriesclassification.com that the script's comments name.", "impact": "minor", "reason": "timeseriesclassification.com returned an HTTP 401 bot challenge to every request from the box (events 12-13); it was not bypassed. The UCR 2018 release is the archive the txt files are derived from, and np.loadtxt parses tab- and space-delimited files identically." }, { "event": 55, "description": "Each dataset ran as a separate process of analysis/bakeoff_one.py, the authors' reproduce_experiments_bakeoff.py with only the dataset-name tuple replaced by one read from an environment variable (complete diff at event 33); kernels, transform, classifier, 10 runs, timers and output are the authors' text. The 85 per-dataset CSVs were merged by analysis/calc.py.", "impact": "none", "reason": "The 2019 script runs all 85 datasets sequentially in one process; one single-core process per dataset, 3 at a time, keeps the paper's single-core timing condition while fitting the time budget." }, { "event": 56, "description": "Timings were measured on a 4-vCPU AMD EPYC 9555P droplet (2024-era CPU) with 3 single-threaded processes running concurrently under a 3.5-CPU quota, rather than on the paper's Xeon E5-2680 v3 / Gold 6150 cluster cores.", "impact": "minor", "reason": "Hardware available for this attempt. Each process had a full core and no memory workaround was needed, but the newer CPU makes the timing claims a check of the paper's upper bound only, not of the exact figures." }, { "event": 57, "description": "The best-mean-rank claim compares the reproduced ROCKET column with the other seven classifiers' published Table 1 accuracies (not re-run), with reproduced ROCKET accuracies rounded half-up to 4 decimals before ranking.", "impact": "none", "reason": "Pre-registered at event 26: the other methods are outside this attempt's scope (days of compute each), and Table 1 reports them at 4 decimals, so ROCKET is compared at the same precision." } ], "notes": [ { "event": 5, "text": "Corrective re-run. This attempt supersedes arxiv-1910-13051-20261017T030206Z (log head fff4af80ae06041663152969f116b4bfeb3611deaf2623e761dc894922bddcf4), which is left unmodified; this is a new, linked version. It was ordered after the independent report audit of 2026-10-19 (office/audits/report-audit-2026-10-19.md, ROCKET section), which found: (1) the earlier attempt never called declare_protocol; (2) it ran the 2022 repository HEAD (36bd908), whose code/rocket_functions.py was rewritten in July 2020 for the journal version, against the numbers in arXiv v1 Table 1; (3) its headline best-mean-rank claim was registered without a tie rule, the reproduced accuracies tied ROCKET with TS-CHIEF exactly (288/85), the earlier summary (artifact 00086) listed ROCKET first, and a tie-averaging rule chosen after seeing the tie (event 90, observed 1.5) made the claim inconsistent and the outcome partially reproduced. This re-run fixes these in order: declare the primary code version (last authors commit on or before the arXiv v1 date) before anything else; register every claim, including the rank claim with its tie rule taken from the paper's own ranking method, before any result; then run all 85 bake-off datasets. Driven by an AI operator (Claude Code subagent); no human in the loop for individual steps. Dedicated cloud box s5-4vcpu-12gb-30gb (atl1), not shared with any other attempt." }, { "event": 34, "text": "Run plan, fixed before any result: all 85 bake-off datasets with the authors code at 4c24610, defaults unchanged (10 runs, 10,000 kernels, RidgeClassifierCV(alphas=10**linspace(-3,3,10), normalize=True)), kernels unseeded as in the authors script. Each dataset runs as its own process of analysis/bakeoff_one.py (the authors script with only the dataset tuple read from an environment variable; full diff at event 33), single-threaded (NUMBA_NUM_THREADS=1 so the authors prange runs on one core; BLAS 1 thread), 3 processes at a time, longest first, via analysis/run_all.py as one background job. Memory: the box has 12 GB (10 GB container limit), enough for the authors in-memory path on every dataset including ElectricDevices, so the earlier attempt's low-memory driver is NOT used and its bit-identity check therefore does not need repeating at this commit. If a process fails it is retried once with the same code; a dataset that fails twice will be reported, not substituted. Data: UCR archive 2018 release from cs.ucr.edu (event 14). Throughput probe on random data (event 16) projects about 39 single-core minutes per run, about 6.4 core-hours for 10 runs, about 2.2 hours wall at 3 processes." }, { "event": 58, "text": "Comparison with the earlier attempt (arxiv-1910-13051-20261017T030206Z, HEAD 36bd908, no declared protocol). Mean accuracy 0.851669 here vs 0.850932 earlier (Table 1 mean 0.851002). Mean absolute per-dataset difference 0.001975 vs 0.002117. Best mean rank: here ROCKET 571/170 vs TS-CHIEF 579/170, so ROCKET is strictly first and the pre-registered tie rule (a tie would count as not reproduced) was never triggered. Earlier, ROCKET and TS-CHIEF tied at 288/85 and the tie rule was chosen after the tie was seen. Both runs use unseeded random kernels, so the two attempts sample different kernels as well as different code versions. The rank result therefore moves with run-to-run variation: one run gives a tie, this one gives a lead of 8/170. This binary check sits at the edge of that noise, as stated when the claim was registered and as the paper says (the top-four differences are not significant). Timing: 40.54 min total vs 111.18 earlier. The earlier figure was inflated by memory pressure and a disk-backed workaround; this one is measured single-core on a newer CPU. ElectricDevices training (2.35 min) is testable here for the first time because the in-memory path fit (peak RSS 7052 MB). The low-memory driver from the earlier attempt was not used, so its bit-identity check was not repeated at this commit." } ], "flags": [], "compute": { "shell_commands": 15, "sandbox_seconds": 56.2, "jobs": 1, "job_seconds": 8184.6, "sandbox": { "container": "repro-arxiv-1910-13051-20261019t203424z", "docker_daemon": { "mem_bytes": 12542218240, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 10737418240 }, "image": "repro-sandbox:0.2", "image_id": "sha256:0f6b339a04afb2effd8987f34d4d8ae2fa5c1049c638ef0daa45a208ee3e3a7a", "limits": { "cpus": 3.5, "gpu": null, "memory": "10g", "network": true, "shm": "2g" }, "target": "cloud", "target_detail": { "box": "arxiv-1910-13051-20261019t203424z", "droplet": "repro-arxiv-1910-13051-20261019-2034", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.11589, "provider": "digitalocean", "region": "atl1", "size": "s5-4vcpu-12gb-30gb" }, "volume": "repro-arxiv-1910-13051-20261019t203424z-work", "warnings": [] }, "environment_event": 60 }, "drivers": [ "operator:claude-opus-5-5 (Claude Code subagent, AI-driven)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-19T20:34:24.315Z", "ended_at": "2026-10-19T23:03:51.101Z", "log": { "events": 63, "head_hash": "68c626f8d6f80233c3b267a4dde2358204dbe838f3ad78eb1893343a1be27175" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }