{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1810-08473-20261018T013946Z", "legacy": true, "paper": { "arxiv_id": "1810.08473", "title": "From Louvain to Leiden: guaranteeing well-connected communities", "authors": [ "V. A. Traag", "L. Waltman", "Nees Jan van Eck" ], "year": 2019, "doi": "10.1038/s41598-019-41695-z", "venue": "Scientific Reports", "openalex_id": "https://openalex.org/W2897249806" }, "code": [ { "commit": "2d385631ccbaffcdf4202e7742ff48341e1792f5", "full_name": "CWTSLeiden/networkanalysis", "modified_files": 0, "path": "/work/networkanalysis", "url": "https://github.com/CWTSLeiden/networkanalysis", "matches_primary": true } ], "code_used": [ { "commit": "2d385631", "repo": null, "code_version": "primary", "source": "method_text", "results": [ 29, 30, 31, 32 ] } ], "primary_code_source": "results", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "broad_bound", "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [ "dblp-leiden-gain: observed 0.938x the published value; passes a registered floor of 0x the published value", "amazon-leiden-gain: observed 1.04x the published value; passes a registered floor of 0x the published value" ], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "Using the authors Java implementation at the release tag matching the paper (networkanalysis 1.0.0) on the SNAP DBLP and Amazon networks, with the paper protocol of 10 random starts of 10 iterations each, all tested Table II values reproduced within 0.0005: DBLP Louvain 0.8267 vs 0.8262 and Leiden 0.8384 vs 0.8387; Amazon Louvain 0.9298 vs 0.9301 and Leiden 0.9340 vs 0.9341. Leiden found higher modularity than Louvain on both networks, as claimed. The other four networks in Table II were out of scope (download blocked, proprietary, or too large for the sandbox), and the badly connected community percentages in Fig. 4 were not tested.", "claims": [ { "claim_id": "dblp-leiden", "comparison": "within", "headline": true, "location": "Table II, p. 6 (text p. 9)", "metric": "max modularity (gamma = 1) over 10 random starts x 10 iterations, Leiden, SNAP com-DBLP", "paper_value": 0.8387, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On DBLP, the maximal modularity found by the Leiden algorithm over 10 replications of 10 iterations is 0.8387.", "tolerance": 0.002, "tolerance_rationale": "Each value is the best of 10 randomized replications of 10 iterations, reported to 4 decimals; the maximum of 10 random starts moves by roughly 1e-3 between independent sets of runs on networks of this size, so 0.002 allows for that and for rounding.", "unit": "modularity", "registered_event": 10, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0023846428997257663, "extraction_registered": false, "results": [ { "event": 29, "observed_value": 0.838377, "delta": -0.0003229999999999622, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Leiden)", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 29, "observed_value": 0.838377, "delta": -0.0003229999999999622, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Leiden)", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9996148801716943, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-leiden", "comparison": "within", "headline": true, "location": "Table II, p. 6 (text p. 9)", "metric": "max modularity (gamma = 1) over 10 random starts x 10 iterations, Leiden, SNAP com-Amazon", "paper_value": 0.9341, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On Amazon, the maximal modularity found by the Leiden algorithm is 0.9341.", "tolerance": 0.002, "tolerance_rationale": "Each value is the best of 10 randomized replications of 10 iterations, reported to 4 decimals; the maximum of 10 random starts moves by roughly 1e-3 between independent sets of runs on networks of this size, so 0.002 allows for that and for rounding.", "unit": "modularity", "registered_event": 11, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0021410983834707206, "extraction_registered": false, "results": [ { "event": 30, "observed_value": 0.934022, "delta": -7.80000000000225e-05, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Leiden)", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 30, "observed_value": 0.934022, "delta": -7.80000000000225e-05, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Leiden)", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9999164971630446, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "dblp-louvain", "comparison": "within", "headline": false, "location": "Table II, p. 6 (text p. 9)", "metric": "max modularity (gamma = 1) over 10 random starts x 10 iterations, Louvain, SNAP com-DBLP", "paper_value": 0.8262, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On DBLP, the maximal modularity found by the Louvain algorithm is 0.8262.", "tolerance": 0.002, "tolerance_rationale": "Each value is the best of 10 randomized replications of 10 iterations, reported to 4 decimals; the maximum of 10 random starts moves by roughly 1e-3 between independent sets of runs on networks of this size, so 0.002 allows for that and for rounding.", "unit": "modularity", "registered_event": 12, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.002420721374969741, "extraction_registered": false, "results": [ { "event": 31, "observed_value": 0.826653, "delta": 0.0004529999999999257, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Louvain)", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 31, "observed_value": 0.826653, "delta": 0.0004529999999999257, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Louvain)", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0005482933914305, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-louvain", "comparison": "within", "headline": false, "location": "Table II, p. 6 (text p. 9)", "metric": "max modularity (gamma = 1) over 10 random starts x 10 iterations, Louvain, SNAP com-Amazon", "paper_value": 0.9301, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On Amazon, the maximal modularity found by the Louvain algorithm is 0.9301.", "tolerance": 0.002, "tolerance_rationale": "Each value is the best of 10 randomized replications of 10 iterations, reported to 4 decimals; the maximum of 10 random starts moves by roughly 1e-3 between independent sets of runs on networks of this size, so 0.002 allows for that and for rounding.", "unit": "modularity", "registered_event": 13, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0021503064186646596, "extraction_registered": false, "results": [ { "event": 32, "observed_value": 0.929846, "delta": -0.0002540000000000875, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Louvain)", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 32, "observed_value": 0.929846, "delta": -0.0002540000000000875, "consistent": true, "evidence_events": [ 22 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 22", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "networkanalysis tag 1.0.0 (commit 2d385631) built with the authors release.sh javac/jar commands; RunNetworkClustering -q modularity -r 1 -s 10 -i 10 --seed 20261018 on the SNAP network; maximum over the 10 random starts as printed by the tool, confirmed by an independent numpy modularity computation on the output clustering. (Louvain)", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9997269110848295, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "dblp-leiden-gain", "comparison": "at_least", "headline": true, "location": "Table II, p. 6 (text p. 9)", "metric": "Leiden max modularity minus Louvain max modularity on DBLP", "paper_value": 0.0125, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On DBLP, Leiden finds higher maximal modularity than Louvain (paper: +0.0125).", "tolerance": 0.0125, "tolerance_rationale": "The paper's comparative claim is that Leiden does at least as well as Louvain; the criterion is a non-negative difference.", "unit": "modularity", "registered_event": 14, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "bound", "broad_bound": true, "tolerance_ratio": 1.0, "extraction_registered": false, "results": [ { "event": 33, "observed_value": 0.011724, "delta": -0.0007760000000000006, "consistent": true, "evidence_events": [ 28 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 28", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Difference of the two DBLP maxima from the job output (event 22), computed by gains.py.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 33, "observed_value": 0.011724, "delta": -0.0007760000000000006, "consistent": true, "evidence_events": [ 28 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 28", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Difference of the two DBLP maxima from the job output (event 22), computed by gains.py.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.93792, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-leiden-gain", "comparison": "at_least", "headline": true, "location": "Table II, p. 6 (text p. 9)", "metric": "Leiden max modularity minus Louvain max modularity on Amazon", "paper_value": 0.004, "quote": "For each network, Table II reports the maximal modularity obtained using the Louvain and the Leiden algorithm.", "statement": "On Amazon, Leiden finds higher maximal modularity than Louvain (paper: +0.0040).", "tolerance": 0.004, "tolerance_rationale": "As for DBLP: the criterion is a non-negative difference.", "unit": "modularity", "registered_event": 15, "quote_verified": true, "quote_page": 9, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "bound", "broad_bound": true, "tolerance_ratio": 1.0, "extraction_registered": false, "results": [ { "event": 34, "observed_value": 0.004176, "delta": 0.00017599999999999994, "consistent": true, "evidence_events": [ 28 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 28", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Difference of the two Amazon maxima from the job output (event 22), computed by gains.py.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 34, "observed_value": 0.004176, "delta": 0.00017599999999999994, "consistent": true, "evidence_events": [ 28 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 28", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": null, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "Difference of the two Amazon maxima from the job output (event 22), computed by gains.py.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.044, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 4, "consistent": 4, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 2, "consistent": 2, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": null, "deviations": [ { "event": 35, "description": "Built and run with OpenJDK 21 instead of a 2018-era JDK.", "impact": "none", "reason": "Current Debian package. The code uses java.util.Random, whose sequence is specified by the Java language and unchanged across versions." }, { "event": 36, "description": "SNAP node ids were renumbered to 0..n-1 in sorted order, and runs used a fixed seed (20261018) rather than an unspecified random one.", "impact": "minor", "reason": "The tool requires zero-indexed ids; the paper does not say how ids were mapped or which seeds were used. Both choices affect only which random starts are drawn, not the method." } ], "notes": [ { "event": 16, "text": "Scope, declared before any data is downloaded or any run made: only DBLP and Amazon from Table II are tested. IMDB (sparse.tamu.edu) returned HTTP 403 when checked during candidate selection, Web of Science is proprietary, and LiveJournal (4M nodes) and Web UK (39M nodes) exceed this sandbox (6 GB). The Fig. 4 percentages of badly connected communities are not tested because the repository has no script for that analysis." }, { "event": 37, "text": "Network sizes match Table II (DBLP 317,080 nodes, mean degree 6.6; Amazon 334,863 nodes). The current SNAP Amazon file gives mean degree 5.53 (925,872 edges), which rounds to 5.5, while Table II lists 5.6. The node count matches exactly, so this is likely rounding or a slightly different edge count in the file version used for the paper; it does not change the comparison." } ], "flags": [], "compute": { "shell_commands": 7, "sandbox_seconds": 7.0, "jobs": 1, "job_seconds": 171.1, "sandbox": { "container": "repro-arxiv-1810-08473-20261018t013946z", "docker_daemon": { "mem_bytes": 8326955008, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 6442450944 }, "image": "repro-sandbox:0.2", "image_id": "sha256:134b91122c639b4bcefe5d3c1c44f6747b122ec731c44eb4c24f4f82cfd0018f", "limits": { "cpus": 3.5, "gpu": null, "memory": "6g", "network": true }, "target": "cloud", "target_detail": { "droplet": "repro-sandbox-20261018-0139", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.08333, "provider": "digitalocean", "region": "nyc3", "size": "s-4vcpu-8gb-intel" }, "volume": "repro-arxiv-1810-08473-20261018t013946z-work", "warnings": [] }, "environment_event": 39 }, "drivers": [ "operator:claude-opus-5-5 (Claude Code session, no human in the loop)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-18T01:39:46.321Z", "ended_at": "2026-10-18T01:48:45.899Z", "log": { "events": 42, "head_hash": "578432e9062b72e8e2576db5eeed0bf5ac96e6f5b649204a2e77964f446edc6c" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }