{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1708-05031-20261019T135218Z", "legacy": true, "paper": { "arxiv_id": "1708.05031", "title": "Neural Collaborative Filtering", "authors": [ "Xiangnan He", "Lizi Liao", "Hanwang Zhang", "Liqiang Nie", "Xia Hu", "Tat-Seng Chua" ], "year": 2017, "doi": null, "venue": null, "openalex_id": "https://openalex.org/W2951707557" }, "code": [ { "commit": "312aff0ee6f4df0ba60041329a74ed29bc7ac4b4", "full_name": "hexiangnan/neural_collaborative_filtering", "modified_files": 0, "path": "/work/ncf", "url": "https://github.com/hexiangnan/neural_collaborative_filtering", "matches_primary": true } ], "code_used": [ { "commit": "312aff0", "repo": null, "code_version": "primary", "source": "method_text", "results": [ 38, 42, 61 ] } ], "primary_code_source": "protocol", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "claims_registered_after_output", "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "The authors Keras code (commit 312aff0, identical in code to the current default branch) was run on MovieLens-1M with the exact README environment (Python 2.7.15, Keras 1.0.7, Theano 0.8.0) on CPU, using the README commands with 8 predictive factors and 20 epochs. All six pre-registered Table 2/3/4 values at 8 factors landed within 0.005 of the paper: NeuMF without pre-training HR@10 0.6848 vs 0.688 and NDCG@10 0.4088 vs 0.410; NeuMF with pre-training (initialised from the authors bundled GMF/MLP weights rather than retrained ones) 0.6798 vs 0.684 and 0.4054 vs 0.403; MLP-3 0.6707 vs 0.671 and 0.3979 vs 0.399. Each value is one run reported at the best-HR epoch, which is the scripts own convention. Mid-attempt the shared cloud droplet was destroyed by another process; the environment and code were rebuilt with the same recipe and only the in-flight MLP run was repeated.", "claims": [ { "claim_id": "neumf-nopre-8-hr10", "comparison": "within", "headline": true, "location": "Table 2 (MovieLens rows, Factors = 8, 'Without Pre-training' HR@10 column), p. 7", "metric": "HR@10 of NeuMF without pre-training, 8 predictive factors, MovieLens-1M, leave-one-out with 99 sampled negatives", "paper_value": 0.688, "quote": "8 0.684 0.403 0.688 0.410", "statement": "NeuMF trained from scratch (without pre-training) with 8 predictive factors reaches HR@10 = 0.688 on MovieLens-1M.", "tolerance": 0.02, "tolerance_rationale": "HR@10 is the mean of 6,040 per-user binary hits, so its binomial standard error is about sqrt(0.69*0.31/6040) = 0.006; run-to-run variation from random initialisation and negative sampling (the paper reports a single run) is of similar size. 0.02 absolute is roughly three combined standard errors and is also the smallest step that separates adjacent factor sizes in the paper's own tables.", "unit": "ratio", "registered_event": 12, "quote_verified": true, "quote_page": 7, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.029069767441860468, "extraction_registered": false, "results": [ { "event": 38, "observed_value": 0.6848, "delta": -0.0031999999999999806, "consistent": true, "evidence_events": [ 34 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 34", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README NeuMF command without pre-training (8 factors, layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001) on the primary commit 312aff0 with Keras 1.0.7/Theano 0.8.0 on CPU; value is the scripts End-of-run best-epoch HR@10 (epoch 18 of 20).", "notes": "Per-epoch HR@10 over the last 6 epochs ranged 0.6738-0.6848; the final epoch (19) gave 0.6844. Single run, single seed.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 38, "observed_value": 0.6848, "delta": -0.0031999999999999806, "consistent": true, "evidence_events": [ 34 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 34", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README NeuMF command without pre-training (8 factors, layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001) on the primary commit 312aff0 with Keras 1.0.7/Theano 0.8.0 on CPU; value is the scripts End-of-run best-epoch HR@10 (epoch 18 of 20).", "notes": "Per-epoch HR@10 over the last 6 epochs ranged 0.6738-0.6848; the final epoch (19) gave 0.6844. Single run, single seed.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9953488372093023, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "neumf-nopre-8-ndcg10", "comparison": "within", "headline": true, "location": "Table 2 (MovieLens rows, Factors = 8, 'Without Pre-training' NDCG@10 column), p. 7", "metric": "NDCG@10 of NeuMF without pre-training, 8 predictive factors, MovieLens-1M", "paper_value": 0.41, "quote": "8 0.684 0.403 0.688 0.410", "statement": "NeuMF trained from scratch (without pre-training) with 8 predictive factors reaches NDCG@10 = 0.410 on MovieLens-1M.", "tolerance": 0.02, "tolerance_rationale": "NDCG@10 averages per-user values in [0,1] over 6,040 users with standard error about 0.006, plus seed/negative-sampling variation of similar size for a single reported run. 0.02 absolute is roughly three combined standard errors and matches the HR tolerance so both metrics are judged on the same footing.", "unit": "ratio", "registered_event": 13, "quote_verified": true, "quote_page": 7, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.04878048780487805, "extraction_registered": false, "results": [ { "event": 39, "observed_value": 0.4088, "delta": -0.0011999999999999789, "consistent": true, "evidence_events": [ 34 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 34", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as neumf-nopre-8-hr10 (job neumf-nopre-8, event 34): NDCG@10 at the best-HR epoch (18) as reported by the scripts End line.", "notes": "The final epoch (19) reached NDCG@10 = 0.4109, slightly higher, but the authors code reports NDCG at the best-HR epoch and that convention is used here. Single run.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 39, "observed_value": 0.4088, "delta": -0.0011999999999999789, "consistent": true, "evidence_events": [ 34 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 34", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as neumf-nopre-8-hr10 (job neumf-nopre-8, event 34): NDCG@10 at the best-HR epoch (18) as reported by the scripts End line.", "notes": "The final epoch (19) reached NDCG@10 = 0.4109, slightly higher, but the authors code reports NDCG at the best-HR epoch and that convention is used here. Single run.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9970731707317073, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "neumf-pre-8-hr10", "comparison": "within", "headline": true, "location": "Table 2 (MovieLens rows, Factors = 8, 'With Pre-training' HR@10 column), p. 7", "metric": "HR@10 of NeuMF with pre-training, 8 predictive factors, MovieLens-1M", "paper_value": 0.684, "quote": "8 0.684 0.403 0.688 0.410", "statement": "NeuMF initialised from pre-trained GMF and MLP (alpha = 0.5) with 8 predictive factors reaches HR@10 = 0.684 on MovieLens-1M.", "tolerance": 0.02, "tolerance_rationale": "HR@10 is the mean of 6,040 per-user binary hits, so its binomial standard error is about sqrt(0.69*0.31/6040) = 0.006; run-to-run variation from random initialisation and negative sampling (the paper reports a single run) is of similar size. 0.02 absolute is roughly three combined standard errors and is also the smallest step that separates adjacent factor sizes in the paper's own tables.", "unit": "ratio", "registered_event": 14, "quote_verified": true, "quote_page": 7, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.029239766081871343, "extraction_registered": false, "results": [ { "event": 42, "observed_value": 0.6798, "delta": -0.0042000000000000925, "consistent": true, "evidence_events": [ 40 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 40", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README NeuMF command with pre-training (8 factors, layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001, alpha 0.5 via the scripts load_pretrain_model) initialised from the repositorys bundled Pretrain/ GMF and MLP weights, primary commit 312aff0, Keras 1.0.7/Theano 0.8.0, CPU; value is the scripts End-of-run best-epoch HR@10 (epoch 9 of 20).", "notes": "Before any fine-tuning (Init line) the pre-trained initialisation already scored HR@10 = 0.6710, NDCG@10 = 0.3969; the final epoch (19) gave 0.6720 / 0.3990. As the paper notes for MovieLens at 8 factors, the pre-trained NeuMF (0.6798) did not beat NeuMF from scratch (0.6848, event 34). Pre-trained weights are the authors files, not retrained here (see deviation at event 26).", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 42, "observed_value": 0.6798, "delta": -0.0042000000000000925, "consistent": true, "evidence_events": [ 40 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 40", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README NeuMF command with pre-training (8 factors, layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001, alpha 0.5 via the scripts load_pretrain_model) initialised from the repositorys bundled Pretrain/ GMF and MLP weights, primary commit 312aff0, Keras 1.0.7/Theano 0.8.0, CPU; value is the scripts End-of-run best-epoch HR@10 (epoch 9 of 20).", "notes": "Before any fine-tuning (Init line) the pre-trained initialisation already scored HR@10 = 0.6710, NDCG@10 = 0.3969; the final epoch (19) gave 0.6720 / 0.3990. As the paper notes for MovieLens at 8 factors, the pre-trained NeuMF (0.6798) did not beat NeuMF from scratch (0.6848, event 34). Pre-trained weights are the authors files, not retrained here (see deviation at event 26).", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9938596491228069, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "neumf-pre-8-ndcg10", "comparison": "within", "headline": true, "location": "Table 2 (MovieLens rows, Factors = 8, 'With Pre-training' NDCG@10 column), p. 7", "metric": "NDCG@10 of NeuMF with pre-training, 8 predictive factors, MovieLens-1M", "paper_value": 0.403, "quote": "8 0.684 0.403 0.688 0.410", "statement": "NeuMF initialised from pre-trained GMF and MLP (alpha = 0.5) with 8 predictive factors reaches NDCG@10 = 0.403 on MovieLens-1M.", "tolerance": 0.02, "tolerance_rationale": "NDCG@10 averages per-user values in [0,1] over 6,040 users with standard error about 0.006, plus seed/negative-sampling variation of similar size for a single reported run. 0.02 absolute is roughly three combined standard errors and matches the HR tolerance so both metrics are judged on the same footing.", "unit": "ratio", "registered_event": 15, "quote_verified": true, "quote_page": 7, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.04962779156327543, "extraction_registered": false, "results": [ { "event": 43, "observed_value": 0.4054, "delta": 0.0023999999999999577, "consistent": true, "evidence_events": [ 40 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 40", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as neumf-pre-8-hr10 (job neumf-pre-8, event 40): NDCG@10 at the best-HR epoch (9) as reported by the scripts End line.", "notes": "Single run; authors bundled pre-trained GMF/MLP weights used as the initialisation.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 43, "observed_value": 0.4054, "delta": 0.0023999999999999577, "consistent": true, "evidence_events": [ 40 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 40", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as neumf-pre-8-hr10 (job neumf-pre-8, event 40): NDCG@10 at the best-HR epoch (9) as reported by the scripts End line.", "notes": "Single run; authors bundled pre-trained GMF/MLP weights used as the initialisation.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.005955334987593, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "mlp3-8-hr10", "comparison": "within", "headline": false, "location": "Table 3 (MovieLens rows, Factors = 8, MLP-3 column), p. 8", "metric": "HR@10 of MLP-3, 8 predictive factors, MovieLens-1M", "paper_value": 0.671, "quote": "8 0.452 0.628 0.655 0.671 0.678", "statement": "MLP with three hidden layers (32->16->8, embedding 16; the README's --layers [64,32,16,8]) reaches HR@10 = 0.671 on MovieLens-1M at 8 predictive factors.", "tolerance": 0.02, "tolerance_rationale": "HR@10 is the mean of 6,040 per-user binary hits, so its binomial standard error is about sqrt(0.69*0.31/6040) = 0.006; run-to-run variation from random initialisation and negative sampling (the paper reports a single run) is of similar size. 0.02 absolute is roughly three combined standard errors and is also the smallest step that separates adjacent factor sizes in the paper's own tables.", "unit": "ratio", "registered_event": 16, "quote_verified": true, "quote_page": 8, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.029806259314456036, "extraction_registered": false, "results": [ { "event": 61, "observed_value": 0.6707, "delta": -0.000300000000000078, "consistent": true, "evidence_events": [ 59 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 59", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README MLP command (layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001) on the primary commit 312aff0 with Keras 1.0.7/Theano 0.8.0 on CPU, re-run on a fresh droplet after the first run was lost (events 44-48); value is the scripts End-of-run best-epoch HR@10 (epoch 18 of 20).", "notes": "Per-epoch HR@10 over the last 6 epochs ranged 0.6573-0.6707; final epoch (19) gave 0.6674. Single run, single seed.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 61, "observed_value": 0.6707, "delta": -0.000300000000000078, "consistent": true, "evidence_events": [ 59 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 59", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "README MLP command (layers [64,32,16,8], 20 epochs, batch 256, 4 negatives, Adam 0.001) on the primary commit 312aff0 with Keras 1.0.7/Theano 0.8.0 on CPU, re-run on a fresh droplet after the first run was lost (events 44-48); value is the scripts End-of-run best-epoch HR@10 (epoch 18 of 20).", "notes": "Per-epoch HR@10 over the last 6 epochs ranged 0.6573-0.6707; final epoch (19) gave 0.6674. Single run, single seed.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.999552906110283, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "mlp3-8-ndcg10", "comparison": "within", "headline": false, "location": "Table 4 (MovieLens rows, Factors = 8, MLP-3 column), p. 9", "metric": "NDCG@10 of MLP-3, 8 predictive factors, MovieLens-1M", "paper_value": 0.399, "quote": "8 0.253 0.359 0.383 0.399 0.406", "statement": "MLP with three hidden layers reaches NDCG@10 = 0.399 on MovieLens-1M at 8 predictive factors.", "tolerance": 0.02, "tolerance_rationale": "NDCG@10 averages per-user values in [0,1] over 6,040 users with standard error about 0.006, plus seed/negative-sampling variation of similar size for a single reported run. 0.02 absolute is roughly three combined standard errors and matches the HR tolerance so both metrics are judged on the same footing.", "unit": "ratio", "registered_event": 17, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": true, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.05012531328320802, "extraction_registered": false, "results": [ { "event": 62, "observed_value": 0.3979, "delta": -0.0011000000000000454, "consistent": true, "evidence_events": [ 59 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 59", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as mlp3-8-hr10 (job mlp3-8-rerun, event 59): NDCG@10 at the best-HR epoch (18) as reported by the scripts End line.", "notes": "Final epoch (19) reached NDCG@10 = 0.3985. Single run.", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 62, "observed_value": 0.3979, "delta": -0.0011000000000000454, "consistent": true, "evidence_events": [ 59 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 59", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Same run as mlp3-8-hr10 (job mlp3-8-rerun, event 59): NDCG@10 at the best-HR epoch (18) as reported by the scripts End line.", "notes": "Final epoch (19) reached NDCG@10 = 0.3985. Single run.", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9972431077694235, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 4, "consistent": 4, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 2, "consistent": 2, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": { "earlier_code": "not_allowed", "primary_code": "github.com/hexiangnan/neural_collaborative_filtering at commit 312aff0 (2017-08-21, 'Change the name of Data'), the last commit on or before the arXiv v2 date 2017-08-26 (v1 was 2017-08-16; the PDF tested is v2). Event 9 shows HEAD 4aab159 differs from 312aff0 only in README.md, LICENSE and Dockerfile, so the code is identical to the current default branch.", "rationale": "The only earlier code is the 2017-03-04 'Raw codes (before clearning)' snapshot and the 2017-08-01/02 cleanup commits; the 2017-08-01 commit 'Fix bug in evaluation' means pre-fix code evaluates differently from what the paper's camera-ready/arXiv numbers describe, so falling back to it would not be a faithful test. Primary code only." }, "deviations": [ { "event": 24, "description": "All training and evaluation run on CPU (Theano device=cpu, 3.5-CPU/6 GB container on a shared 4-vCPU DigitalOcean droplet, OMP_NUM_THREADS=1), not on whatever hardware the authors used.", "impact": "none", "reason": "The engine is CPU-only. The paper does not state its hardware; the models are small (8 predictive factors) and the authors code contains no GPU-specific logic, so the device changes speed, not the computation." }, { "event": 25, "description": "Python 2.7.15 with Keras 1.0.7 and Theano 0.8.0 exactly as the README specifies, but with numpy 1.16.5, scipy 1.2.1 and h5py 2.10.0 (the last Python 2 releases from conda-forge/PyPI) rather than the unspecified 2017-era versions; Theano compiles with the sandbox system g++ 14.2.", "impact": "none", "reason": "The README pins only Keras and Theano; those pins installed exactly. The scientific-stack versions are not stated anywhere in the paper or repository." }, { "event": 26, "description": "For the NeuMF with pre-training claims, the pre-trained GMF (8 factors) and MLP ([64,32,16,8]) weights are the authors files bundled in the repository (Pretrain/ml-1m_GMF_8_1501651698.h5 and Pretrain/ml-1m_MLP_[64,32,16,8]_1501652038.h5, as named in the README command) rather than GMF and MLP models retrained in this attempt.", "impact": "minor", "reason": "Time budget: retraining GMF and MLP first (about 45-70 min each on CPU) and then NeuMF would exceed the 2-3 hour window with at most two concurrent training processes on the shared box. The README with-pre-training command itself uses these files. The with-pre-training result therefore tests the NeuMF fine-tuning stage on the authors initialisation, not the full GMF->MLP->NeuMF pipeline." }, { "event": 48, "description": "The cloud sandbox was destroyed mid-attempt by another process (shared droplet torn down at ~15:17Z) and rebuilt on a fresh droplet: the Python 2.7 / Keras 1.0.7 / Theano 0.8.0 environment and the authors repository were set up a second time with the same recipe and commit, and the MLP-3 run (job mlp3-8) was restarted from scratch. The NeuMF results were complete before the loss and were not re-run.", "impact": "minor", "reason": "Operator error on a shared box, not anything in the paper or code. The environment recipe is deterministic (pinned versions), so the second setup is equivalent; the restarted MLP run is a fresh single run like the original would have been." } ], "notes": [ { "event": 27, "text": "Plan: GMF timing (event 21) gives ~128 s/epoch train + ~10 s eval at 8 factors, so 20 epochs of GMF is ~46 min and NeuMF/MLP are slower. With at most two concurrent training processes on the shared droplet, the order is: NeuMF without pre-training and NeuMF with pre-training now (20 epochs each, README commands), then MLP-3 (README MLP command) when a slot frees. GMF at 8 factors has no table value in the paper (Figure 4 only) and is not a registered claim. Observed values will be taken from the scripts own End-of-run line (best epoch by HR@10 over the 20 evaluated epochs), which is how the authors code reports a run; the paper does not say whether Table 2 reports the last or the best iteration." }, { "event": 47, "text": "Infrastructure incident: the shared DigitalOcean droplet (repro-sandbox-20261019-1321, hosting this attempts container and its /work volume alongside another attempts container) was destroyed by another process (the other attempts operator ran repro cloud down) at about 15:17Z, while job mlp3-8 was at iteration ~12 of 20. doctl now lists no droplets and SSH to the box times out (event 44 error, this note). The sandbox, the /env/py27 environment, the cloned code and the in-flight MLP job were lost. Results recorded before this (events 34 and 40, claims at events 38, 39, 42, 43) were produced and logged before the loss and remain valid evidence. Plan: bring up a fresh box, rebuild the environment with the same recipe, re-clone the primary commit and re-run only the lost MLP-3 job within the remaining budget." } ], "flags": [], "compute": { "shell_commands": 5, "sandbox_seconds": 212.3, "jobs": 5, "job_seconds": 7893.6, "sandbox": { "container": "repro-arxiv-1708-05031-20261019t135218z", "docker_daemon": { "mem_bytes": 8326955008, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 6442450944 }, "image": "repro-sandbox:0.2", "image_id": "sha256:8601474b6bbfa9a7eef6b9c1226a9d6dfcc2e1cf81aad13ff2c65a8e8b1313c9", "limits": { "cpus": 3.5, "gpu": null, "memory": "6g", "network": true, "shm": "2g" }, "target": "cloud", "target_detail": { "droplet": "repro-sandbox-20261019-1321", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.08333, "provider": "digitalocean", "region": "nyc3", "size": "s-4vcpu-8gb-intel" }, "volume": "repro-arxiv-1708-05031-20261019t135218z-work", "warnings": [] }, "environment_event": 64 }, "drivers": [ "operator:claude (Claude Code subagent, AI-driven)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-19T13:52:18.131Z", "ended_at": "2026-10-19T16:00:36.780Z", "log": { "events": 67, "head_hash": "04ac3824830007550ce23545b33c04ec2f8c290dd7107cca03355bd8ce4ddfc6" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }