{ "schema": "repro.report/v3", "engine_version": "0.2.1", "attempt_id": "arxiv-1607-01759-20261018T223713Z", "legacy": true, "paper": { "arxiv_id": "1607.01759", "title": "Bag of Tricks for Efficient Text Classification", "authors": [ "Armand Joulin", "Édouard Grave", "Piotr Bojanowski", "Tomáš Mikolov" ], "year": 2017, "doi": "10.18653/v1/e17-2068", "venue": "Conference of the European Chapter of the Association for Computational Linguistics", "openalex_id": "https://openalex.org/W2468328197" }, "code": [ { "commit": "206179d64c1730862328e9b750e98bd8aa1c16b5", "full_name": "facebookresearch/fastText", "modified_files": 0, "path": "/work/fastText", "url": "https://github.com/facebookresearch/fastText", "matches_primary": true } ], "code_used": [ { "commit": "206179d", "repo": null, "code_version": "primary", "source": "method_text", "results": [ 33, 41, 34, 42, 35, 43, 36, 44, 37, 45, 38, 46, 39, 47, 40, 48 ] } ], "primary_code_source": "results", "outcome": "reproduced", "outcome_label": "Reproduced", "qualifiers": [ "plan_not_sealed", "evidence_not_bound", "agent_selected_extraction" ], "qualifier_details": [], "evidence_gate_failed": [], "assurance": { "plan_sealed": false, "sealed_before_scored_execution": false, "evidence_bound": false, "extraction_sealed": false, "execution_manifest": false, "semantic_rederivation_passed": true, "predates": [ "plan_seal", "evidence_binding", "execution_manifest", "paper_text_hash", "scored_job_flag", "retry_reasons", "extraction_rule" ], "engine_version": "0.2.1", "attempt_engine_version": "0.1.0", "setup_outputs_before_seal": null, "observations_before_seal": null, "observation_provenance": null, "plan_hash": null, "paper_text_sha256_ok": null }, "plan": null, "not_attemptable_reason": null, "summary": "With fastText at the commit of arXiv v3 and its classification-results.sh, test accuracy with 10 hidden units and bigrams reproduces Table 1 on all eight datasets: 92.2, 96.7, 98.6, 95.7, 63.9, 72.4, 60.2 and 94.6 against 92.5, 96.8, 98.6, 95.7, 63.9, 72.3, 60.2 and 94.6. The unigram row also reproduces, within 0.2 points everywhere. The script own Google Drive download no longer works, so the datasets came from a public mirror of the same tarballs, preprocessed with the script own function. Training speed (Table 2) was not tested.", "claims": [ { "claim_id": "ag-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], AG News, fastText h=10 with bigrams", "paper_value": 92.5, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 92.5% test accuracy on AG News.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 10, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003243243243243243, "extraction_registered": false, "results": [ { "event": 33, "observed_value": 92.2, "delta": -0.29999999999999716, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 33, "observed_value": 92.2, "delta": -0.29999999999999716, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9967567567567568, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "ag-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], AG News, fastText h=10 without bigrams", "paper_value": 91.5, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 91.5% test accuracy on AG News.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 11, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003278688524590164, "extraction_registered": false, "results": [ { "event": 41, "observed_value": 91.4, "delta": -0.09999999999999432, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 41, "observed_value": 91.4, "delta": -0.09999999999999432, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9989071038251367, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "sogou-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Sogou News, fastText h=10 with bigrams", "paper_value": 96.8, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 96.8% test accuracy on Sogou News.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 12, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0030991735537190084, "extraction_registered": false, "results": [ { "event": 34, "observed_value": 96.7, "delta": -0.09999999999999432, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 34, "observed_value": 96.7, "delta": -0.09999999999999432, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9989669421487604, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "sogou-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Sogou News, fastText h=10 without bigrams", "paper_value": 93.9, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 93.9% test accuracy on Sogou News.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 13, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003194888178913738, "extraction_registered": false, "results": [ { "event": 42, "observed_value": 93.7, "delta": -0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 42, "observed_value": 93.7, "delta": -0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9978700745473909, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "dbpedia-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], DBpedia, fastText h=10 with bigrams", "paper_value": 98.6, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 98.6% test accuracy on DBpedia.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 14, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0030425963488843813, "extraction_registered": false, "results": [ { "event": 35, "observed_value": 98.6, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 35, "observed_value": 98.6, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "dbpedia-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], DBpedia, fastText h=10 without bigrams", "paper_value": 98.1, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 98.1% test accuracy on DBpedia.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 15, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0030581039755351682, "extraction_registered": false, "results": [ { "event": 43, "observed_value": 98.3, "delta": 0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 43, "observed_value": 98.3, "delta": 0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0020387359836902, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yelp-p-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yelp Polarity, fastText h=10 with bigrams", "paper_value": 95.7, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 95.7% test accuracy on Yelp Polarity.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 16, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003134796238244514, "extraction_registered": false, "results": [ { "event": 36, "observed_value": 95.7, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 36, "observed_value": 95.7, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yelp-p-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yelp Polarity, fastText h=10 without bigrams", "paper_value": 93.8, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 93.8% test accuracy on Yelp Polarity.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 17, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0031982942430703624, "extraction_registered": false, "results": [ { "event": 44, "observed_value": 93.9, "delta": 0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 44, "observed_value": 93.9, "delta": 0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0010660980810235, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yelp-f-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yelp Full, fastText h=10 with bigrams", "paper_value": 63.9, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 63.9% test accuracy on Yelp Full.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 18, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.004694835680751174, "extraction_registered": false, "results": [ { "event": 37, "observed_value": 63.9, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 37, "observed_value": 63.9, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yelp-f-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yelp Full, fastText h=10 without bigrams", "paper_value": 60.4, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 60.4% test accuracy on Yelp Full.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 19, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.004966887417218543, "extraction_registered": false, "results": [ { "event": 45, "observed_value": 60.3, "delta": -0.10000000000000142, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 45, "observed_value": 60.3, "delta": -0.10000000000000142, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9983443708609271, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yahoo-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yahoo! Answers, fastText h=10 with bigrams", "paper_value": 72.3, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 72.3% test accuracy on Yahoo! Answers.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 20, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.004149377593360996, "extraction_registered": false, "results": [ { "event": 38, "observed_value": 72.4, "delta": 0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 38, "observed_value": 72.4, "delta": 0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0013831258644539, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "yahoo-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Yahoo! Answers, fastText h=10 without bigrams", "paper_value": 72.0, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 72.0% test accuracy on Yahoo! Answers.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 21, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.004166666666666667, "extraction_registered": false, "results": [ { "event": 46, "observed_value": 71.8, "delta": -0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 46, "observed_value": 71.8, "delta": -0.20000000000000284, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9972222222222222, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-f-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Amazon Full, fastText h=10 with bigrams", "paper_value": 60.2, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 60.2% test accuracy on Amazon Full.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 22, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.0049833887043189366, "extraction_registered": false, "results": [ { "event": 39, "observed_value": 60.2, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 39, "observed_value": 60.2, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-f-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Amazon Full, fastText h=10 without bigrams", "paper_value": 55.8, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 55.8% test accuracy on Amazon Full.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 23, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.005376344086021506, "extraction_registered": false, "results": [ { "event": 47, "observed_value": 55.6, "delta": -0.19999999999999574, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 47, "observed_value": 55.6, "delta": -0.19999999999999574, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9964157706093191, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-p-bigram", "comparison": "within", "headline": true, "location": "Table 1, p. 3", "metric": "test accuracy [%], Amazon Polarity, fastText h=10 with bigrams", "paper_value": 94.6, "quote": "fastText, h = 10, bigram 92.5 96.8 98.6 95.7 63.9 72.3 60.2 94.6", "statement": "fastText with 10 hidden units and bigrams reaches 94.6% test accuracy on Amazon Polarity.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 24, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003171247357293869, "extraction_registered": false, "results": [ { "event": 40, "observed_value": 94.6, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 40, "observed_value": 94.6, "delta": 0.0, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 1.0, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true }, { "claim_id": "amazon-p-unigram", "comparison": "within", "headline": false, "location": "Table 1, p. 3", "metric": "test accuracy [%], Amazon Polarity, fastText h=10 without bigrams", "paper_value": 91.2, "quote": "fastText, h = 10 91.5 93.9 98.1 93.8 60.4 72.0 55.8 91.2", "statement": "fastText with 10 hidden units without bigrams reaches 91.2% test accuracy on Amazon Polarity.", "tolerance": 0.3, "tolerance_rationale": "Hogwild multithreaded SGD makes accuracy vary by about 0.1 point between runs, and the code version and data copy may differ slightly; values are reported to 0.1, so 0.3 points is used.", "unit": "percent", "registered_event": 25, "quote_verified": true, "quote_page": 3, "preregistered": true, "registered_after_output": false, "sealed": false, "check_kind": "equivalence", "broad_bound": false, "tolerance_ratio": 0.003289473684210526, "extraction_registered": false, "results": [ { "event": 48, "observed_value": 91.1, "delta": -0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" } ], "withdrawn_after_seal": false, "final": { "event": 48, "observed_value": 91.1, "delta": -0.10000000000000853, "consistent": true, "evidence_events": [ 31 ], "evidence": null, "evidence_binding": "legacy", "evidence_verified": true, "evidence_reason": "value matched by the legacy rule in events 31", "evidence_after_seal": null, "evidence_from_unscored_job": false, "evidence_scored_execution": null, "extraction": "legacy", "job_start_event": null, "tree_modified_during_job": null, "manifest_complete": false, "retry_reason": null, "rerun_rule_checked": false, "rederivation_problems": [], "method": "fastText at commit 206179d (2016-08-09) built with make; classification-results.sh unchanged for bigrams (-dim 10 -wordNgrams 2 -epoch 5 -thread 4, per-dataset learning rates from the script), the same command with -wordNgrams 1 for the unigram row; test P@1 printed by fasttext test, as a percentage.", "notes": "", "code_version": "primary", "code_ref": "" }, "earlier_final": null, "verdict": "consistent", "result_revisions": 0, "ratio_to_published": 0.9989035087719297, "optional_stopping": false, "evidence_problems": [], "evidence_ok": true } ], "counts": { "claims_scored": 8, "consistent": 8, "consistent_earlier_code": 0, "inconsistent": 0, "untested": 0, "withdrawn_after_seal": 0, "other": { "claims": 8, "consistent": 8, "inconsistent": 0, "untested": 0 } }, "withdrawn": [], "protocol": { "earlier_code": "allowed", "primary_code": "github.com/facebookresearch/fastText (linked in the paper) at the last commit on or before 2016-08-09 (arXiv v3), building with make and running classification-results.sh, whose header says it produces Table 1", "rationale": "arXiv v3 is the version tested. Data: the script own Google Drive download is used if it still works; otherwise the public s3.amazonaws.com/fast-ai-nlp mirror of the same dataset tarballs (declared here, before any download). Earlier commits may be tried as a declared fallback for misses." }, "deviations": [ { "event": 49, "description": "Datasets came from the public s3.amazonaws.com/fast-ai-nlp mirror (tarball hashes in the job output) instead of the script own googledrive.com/host download, then preprocessed with the script own normalize_text function.", "impact": "minor", "reason": "googledrive.com/host returned HTTP 404 (event 31); this fallback was declared in the protocol before any download. Train/test sizes match the standard splits (e.g. AG 120,000/7,600)." }, { "event": 50, "description": "The unigram row was produced with the same command as the script but -wordNgrams 1; the script itself only runs the bigram configuration.", "impact": "none", "reason": "The paper reports fastText with and without bigrams under the same parameters." }, { "event": 51, "description": "Built with g++ 14 on Debian 13 and trained with 4 threads on a 4 vCPU shared droplet.", "impact": "minor", "reason": "Current toolchain; Hogwild training is nondeterministic across thread schedules, which the tolerance allows for." } ], "notes": [], "flags": [], "compute": { "shell_commands": 2, "sandbox_seconds": 3.6, "jobs": 1, "job_seconds": 3810.0, "sandbox": { "container": "repro-arxiv-1607-01759-20261018t223713z", "docker_daemon": { "mem_bytes": 8326946816, "ncpu": 4 }, "effective": { "cpus": 3.5, "memory_bytes": 6442450944 }, "image": "repro-sandbox:0.2", "image_id": "sha256:f4b241036e33f995f431b595fba27a8b03cf1f26e83ee872595f9957605f656b", "limits": { "cpus": 3.5, "gpu": null, "memory": "6g", "network": true, "shm": "2g" }, "target": "cloud", "target_detail": { "droplet": "repro-sandbox-20261018-2235", "image": "ubuntu-24-04-x64", "price_hourly_usd": 0.08333, "provider": "digitalocean", "region": "nyc3", "size": "s-4vcpu-8gb-intel" }, "volume": "repro-arxiv-1607-01759-20261018t223713z-work", "warnings": [] }, "environment_event": 53 }, "drivers": [ "operator:claude-opus-5-5 (Claude Code session, no human in the loop)" ], "model_usage": {}, "model_cost_usd": null, "started_at": "2026-10-18T22:37:13.283Z", "ended_at": "2026-10-18T23:44:49.124Z", "log": { "events": 56, "head_hash": "87b5ac6197ec5f210cb74ba6aaa5412fe8e2237bdf845f1afee95be9043de61d" }, "right_of_reply": "Authors may submit a response, which is published alongside this report unedited." }