{ "description": "Controlled data-quality experiments; not a comparison with frontier models or a before/after validation test.", "source_revision": "1fcefb66637638ead2c50ab60d49ffaada57d769", "source_repository": "Thatayotlhe04/a-parallel-paradigm", "author": "Thatayotlhe Tsenang", "date": "2026-11-08", "accuracy": { "baselines": { "clean_exact_match": [ 1.0, 1.0, 1.0, 5 ], "clean_exact_match_sampled": [ 0.9826666666666666, 0.9754088600155582, 0.989924473317775, 5 ], "clean_empty_rate_sampled": [ 0.0, 0.0, 0.0, 5 ], "clean_gold_nll": [ 0.0015108491992577911, 0.0014311601529985436, 0.0015905382455170386, 5 ], "lowdata_clean_exact_match": [ 0.1, 0.04420181864810306, 0.15579818135189694, 5 ], "lowdata_clean_exact_match_sampled": [ 0.07166666666666667, 0.028679062751446834, 0.11465427058188651, 5 ], "lowdata_clean_empty_rate_sampled": [ 0.0, 0.0, 0.0, 5 ], "lowdata_clean_gold_nll": [ 0.7823761463165283, 0.6250337275937919, 0.9397185650392647, 5 ] }, "conditions": [ { "condition": "contamination", "rate": 0.1, "exact_match_mean": 0.9996666666666666, "exact_match_diff": -0.000333333333333341, "exact_match_lo": -0.0012588150350659603, "exact_match_hi": 0.0005921483683992781, "n_seeds": 5, "exact_match_p": 0.37390096630005887, "exact_match_sampled_mean": 0.985, "exact_match_sampled_diff": 0.0023333333333333426, "exact_match_sampled_lo": -0.0038056177766398478, "exact_match_sampled_hi": 0.008472284443306534, "exact_match_sampled_p": 0.35081481481480736, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.001551277027465403, "gold_nll_diff": 4.042782820761204e-05, "gold_nll_lo": -5.654838305662215e-05, "gold_nll_hi": 0.00013740403947184624, "gold_nll_p": 0.31149497509042623, "inflation_mean": 3.7037037037035424e-05, "inflation_lo": -6.579426315547098e-05, "inflation_hi": 0.00013986833722954181, "acc_seen_mean": 1.0, "acc_unseen_mean": 0.9996296296296296 }, { "condition": "contamination", "rate": 0.3, "exact_match_mean": 1.0, "exact_match_diff": 0.0, "exact_match_lo": 0.0, "exact_match_hi": 0.0, "n_seeds": 5, "exact_match_p": 1.0, "exact_match_sampled_mean": 0.9869999999999999, "exact_match_sampled_diff": 0.004333333333333323, "exact_match_sampled_lo": -0.0008609198350844841, "exact_match_sampled_hi": 0.00952758650175113, "exact_match_sampled_p": 0.08146997954982481, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.0014879752648994326, "gold_nll_diff": -2.2873934358358384e-05, "gold_nll_lo": -7.866336611608487e-05, "gold_nll_hi": 3.291549739936809e-05, "gold_nll_p": 0.31852463990176494, "inflation_mean": 0.0, "inflation_lo": 0.0, "inflation_hi": 0.0, "acc_seen_mean": 1.0, "acc_unseen_mean": 1.0 }, { "condition": "dup_concentrated", "rate": 0.1, "exact_match_mean": 1.0, "exact_match_diff": 0.0, "exact_match_lo": 0.0, "exact_match_hi": 0.0, "n_seeds": 5, "exact_match_p": 1.0, "exact_match_sampled_mean": 0.9793333333333333, "exact_match_sampled_diff": -0.0033333333333333435, "exact_match_sampled_lo": -0.009186593553900491, "exact_match_sampled_hi": 0.0025199268872338034, "exact_match_sampled_p": 0.18900365845517003, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.0015205215429887175, "gold_nll_diff": 9.672343730926514e-06, "gold_nll_lo": -8.007319453145817e-05, "gold_nll_hi": 9.94178819933112e-05, "gold_nll_p": 0.7796663824105573 }, { "condition": "dup_concentrated", "rate": 0.3, "exact_match_mean": 1.0, "exact_match_diff": 0.0, "exact_match_lo": 0.0, "exact_match_hi": 0.0, "n_seeds": 5, "exact_match_p": 1.0, "exact_match_sampled_mean": 0.9843333333333334, "exact_match_sampled_diff": 0.001666666666666683, "exact_match_sampled_lo": -0.002723278498758718, "exact_match_sampled_hi": 0.006056611832092084, "exact_match_sampled_p": 0.35130100296325456, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.0017274056794121862, "gold_nll_diff": 0.0002165564801543951, "gold_nll_lo": 0.00013427969659451482, "gold_nll_hi": 0.0002988332637142754, "gold_nll_p": 0.001864914504957384 }, { "condition": "empty_target", "rate": 0.1, "exact_match_mean": 0.954, "exact_match_diff": -0.046, "exact_match_lo": -0.059379533753855164, "exact_match_hi": -0.032620466246144834, "n_seeds": 5, "exact_match_p": 0.0006726708053639232, "exact_match_sampled_mean": 0.8996666666666666, "exact_match_sampled_diff": -0.08300000000000003, "exact_match_sampled_lo": -0.09721751967598709, "exact_match_sampled_hi": -0.06878248032401298, "exact_match_sampled_p": 8.476895894487632e-05, "empty_rate_sampled_mean": 0.07833333333333334, "empty_rate_sampled_diff": 0.07833333333333334, "empty_rate_sampled_lo": 0.06362719903464455, "empty_rate_sampled_hi": 0.09303946763202213, "empty_rate_sampled_p": 0.00012169684452001326, "gold_nll_mean": 0.014243058860301971, "gold_nll_diff": 0.01273220966104418, "gold_nll_lo": 0.009413307600520542, "gold_nll_hi": 0.01605111172156782, "gold_nll_p": 0.0004400054660178182 }, { "condition": "empty_target", "rate": 0.3, "exact_match_mean": 0.776, "exact_match_diff": -0.22399999999999998, "exact_match_lo": -0.26004035364194955, "exact_match_hi": -0.1879596463580504, "n_seeds": 5, "exact_match_p": 6.617563236548885e-05, "exact_match_sampled_mean": 0.6886666666666666, "exact_match_sampled_diff": -0.29400000000000004, "exact_match_sampled_lo": -0.3168389683560204, "exact_match_sampled_hi": -0.2711610316439797, "exact_match_sampled_p": 3.658038719254428e-06, "empty_rate_sampled_mean": 0.284, "empty_rate_sampled_diff": 0.284, "empty_rate_sampled_lo": 0.2645538697454536, "empty_rate_sampled_hi": 0.30344613025454636, "empty_rate_sampled_p": 2.2105248253114557e-06, "gold_nll_mean": 0.05256788358092308, "gold_nll_diff": 0.05105703438166529, "gold_nll_lo": 0.03571747853599316, "gold_nll_hi": 0.06639659022733742, "gold_nll_p": 0.00076218380780538 }, { "condition": "label_noise_random", "rate": 0.1, "exact_match_mean": 0.9886666666666667, "exact_match_diff": -0.011333333333333329, "exact_match_lo": -0.017437304380161892, "exact_match_hi": -0.0052293622865047665, "n_seeds": 5, "exact_match_p": 0.006720880690790092, "exact_match_sampled_mean": 0.908, "exact_match_sampled_diff": -0.07466666666666669, "exact_match_sampled_lo": -0.08762341049092304, "exact_match_sampled_hi": -0.06170992284241033, "exact_match_sampled_p": 8.92165423810604e-05, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.010199067182838917, "gold_nll_diff": 0.008688217983581126, "gold_nll_lo": 0.007877446334010921, "gold_nll_hi": 0.009498989633151331, "gold_nll_p": 7.599806851539109e-06 }, { "condition": "label_noise_random", "rate": 0.3, "exact_match_mean": 0.9630000000000001, "exact_match_diff": -0.03699999999999999, "exact_match_lo": -0.04827797333149523, "exact_match_hi": -0.025722026668504754, "n_seeds": 5, "exact_match_p": 0.0008057456843671433, "exact_match_sampled_mean": 0.7546666666666667, "exact_match_sampled_diff": -0.22800000000000004, "exact_match_sampled_lo": -0.24515264925931504, "exact_match_sampled_hi": -0.21084735074068503, "exact_match_sampled_p": 3.218530169492178e-06, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.03178755827248096, "gold_nll_diff": 0.030276709073223173, "gold_nll_lo": 0.028179738348850715, "gold_nll_hi": 0.03237367979759563, "gold_nll_p": 2.313822981690895e-06 }, { "condition": "label_noise_systematic", "rate": 0.1, "exact_match_mean": 0.8780000000000001, "exact_match_diff": -0.12199999999999997, "exact_match_lo": -0.13838651534664476, "exact_match_hi": -0.10561348465335518, "n_seeds": 5, "exact_match_p": 3.235610872143166e-05, "exact_match_sampled_mean": 0.8646666666666667, "exact_match_sampled_diff": -0.11800000000000002, "exact_match_sampled_lo": -0.13540053276240419, "exact_match_sampled_hi": -0.10059946723759586, "exact_match_sampled_p": 4.685906526376709e-05, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.13127212077379227, "gold_nll_diff": 0.1297612715745345, "gold_nll_lo": 0.10067908648709119, "gold_nll_hi": 0.15884345666197777, "gold_nll_p": 0.00024405537481082666, "acc_on_affected_mean": 0.0, "rows_affected_share_mean": 0.1214 }, { "condition": "label_noise_systematic", "rate": 0.3, "exact_match_mean": 0.6833333333333333, "exact_match_diff": -0.3166666666666667, "exact_match_lo": -0.33070229160664105, "exact_match_hi": -0.30263104172669236, "n_seeds": 5, "exact_match_p": 3.890227521280751e-07, "exact_match_sampled_mean": 0.673, "exact_match_sampled_diff": -0.30966666666666665, "exact_match_sampled_lo": -0.3216801178806498, "exact_match_sampled_hi": -0.2976532154526835, "exact_match_sampled_p": 2.2841422074289615e-07, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.3342846155166626, "gold_nll_diff": 0.3327737663174048, "gold_nll_lo": 0.26429083467448483, "gold_nll_hi": 0.40125669796032476, "gold_nll_p": 0.00017465712824587116, "acc_on_affected_mean": 0.0, "rows_affected_share_mean": 0.3194 }, { "condition": "lowdata_contamination", "rate": 0.1, "exact_match_mean": 0.19966666666666666, "exact_match_diff": 0.09966666666666665, "exact_match_lo": 0.027585959382767505, "exact_match_hi": 0.1717473739505658, "n_seeds": 5, "exact_match_p": 0.018475062571887985, "exact_match_sampled_mean": 0.17266666666666666, "exact_match_sampled_diff": 0.101, "exact_match_sampled_lo": 0.031780402839419866, "exact_match_sampled_hi": 0.17021959716058016, "exact_match_sampled_p": 0.015460687335405648, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.6987105548381806, "gold_nll_diff": -0.08366559147834778, "gold_nll_lo": -0.23433812586326486, "gold_nll_hi": 0.06700694290656932, "gold_nll_p": 0.1980029483892812, "inflation_mean": 0.08892592592592592, "inflation_lo": 0.07877032629394702, "inflation_hi": 0.09908152555790482, "acc_seen_mean": 1.0, "acc_unseen_mean": 0.11074074074074074 }, { "condition": "lowdata_contamination", "rate": 0.3, "exact_match_mean": 0.3496666666666667, "exact_match_diff": 0.24966666666666665, "exact_match_lo": 0.19553587564337363, "exact_match_hi": 0.3037974576899597, "n_seeds": 5, "exact_match_p": 0.00021432737763210232, "exact_match_sampled_mean": 0.33199999999999996, "exact_match_sampled_diff": 0.26033333333333336, "exact_match_sampled_lo": 0.21836909048567463, "exact_match_sampled_hi": 0.30229757618099207, "exact_match_sampled_p": 6.66648087091266e-05, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.6090388417243957, "gold_nll_diff": -0.17333730459213256, "gold_nll_lo": -0.34431382713467795, "gold_nll_hi": -0.0023607820495871457, "gold_nll_p": 0.04808250317284293, "inflation_mean": 0.2787142857142857, "inflation_lo": 0.26881031653704063, "inflation_hi": 0.28861825489153076, "acc_seen_mean": 1.0, "acc_unseen_mean": 0.07095238095238096 }, { "condition": "truncated_target", "rate": 0.1, "exact_match_mean": 1.0, "exact_match_diff": 0.0, "exact_match_lo": 0.0, "exact_match_hi": 0.0, "n_seeds": 5, "exact_match_p": 1.0, "exact_match_sampled_mean": 0.985, "exact_match_sampled_diff": 0.0023333333333333205, "exact_match_sampled_lo": -0.0032579856415444167, "exact_match_sampled_hi": 0.007924652308211058, "exact_match_sampled_p": 0.3110607673343719, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.0015635509276762604, "gold_nll_diff": 5.270172841846943e-05, "gold_nll_lo": 2.3081422385644075e-05, "gold_nll_hi": 8.232203445129479e-05, "gold_nll_p": 0.007816943000957694 }, { "condition": "truncated_target", "rate": 0.3, "exact_match_mean": 0.9996666666666666, "exact_match_diff": -0.000333333333333341, "exact_match_lo": -0.0012588150350659603, "exact_match_hi": 0.0005921483683992781, "n_seeds": 5, "exact_match_p": 0.37390096630005887, "exact_match_sampled_mean": 0.9823333333333334, "exact_match_sampled_diff": -0.000333333333333341, "exact_match_sampled_lo": -0.006075796275252183, "exact_match_sampled_hi": 0.005409129608585501, "exact_match_sampled_p": 0.8797762036739447, "empty_rate_sampled_mean": 0.0, "empty_rate_sampled_diff": 0.0, "empty_rate_sampled_lo": 0.0, "empty_rate_sampled_hi": 0.0, "empty_rate_sampled_p": 1.0, "gold_nll_mean": 0.0018418311607092618, "gold_nll_diff": 0.00033098196145147084, "gold_nll_lo": -3.210392072424458e-06, "gold_nll_hi": 0.0006651743149753661, "gold_nll_p": 0.05138639509539451 } ] }, "detection": { "meta": { "alpaca_sha256": "bd844b8247a0f543804b6ce0882b0aaec4bbf5e8d66167df6213a0f1e4fe878b", "hh_sha256": "518a5bf288456fc9f3b7c980c54116fba0c52f274d3f4d344675d83e4058f6f4", "alpaca_rows": 51760, "hh_pairs_parsed": 43709, "seed": 20261108, "natural_full_alpaca": { "seconds": 6.39, "rows_per_sec": 8104, "rows_with_error": 4, "rows_with_any": 61, "by_rule": { "duplicates": 4, "pii": 57 } }, "base_raw_flags": { "parallelogram": 13, "cookbook": 0, "exact": 0, "minhash": 0, "ngram13": 24 }, "base_clean_rows": 9963, "base_clean_flags": { "parallelogram": 0, "cookbook": 0, "exact": 0, "minhash": 0, "ngram13": 0 }, "pref_raw_flags": { "parallelogram": 28, "exact": 0, "minhash": 12, "parallelogram_by_rule": { "preference": 7, "empty-content": 9, "roles": 5, "pii": 6, "encoding": 1 } }, "pref_clean_rows": 4960, "fix": { "summary": { "total": 9963, "unchanged": 9813, "fixed": 50, "dropped": 90, "unparseable": 10, "emitted": 9863 }, "clean_rows": 9773, "clean_rows_kept_unchanged": 9768, "injected_fate": { "invalid_json": { "kept_unchanged": 0, "removed_or_changed": 10 }, "missing_assistant": { "kept_unchanged": 0, "removed_or_changed": 10 }, "role_order": { "kept_unchanged": 0, "removed_or_changed": 10 }, "empty_response": { "kept_unchanged": 0, "removed_or_changed": 10 }, "whitespace_response": { "kept_unchanged": 0, "removed_or_changed": 10 }, "overlong_response": { "kept_unchanged": 0, "removed_or_changed": 10 }, "overlong_user": { "kept_unchanged": 0, "removed_or_changed": 10 }, "secret_aws": { "kept_unchanged": 0, "removed_or_changed": 10 }, "secret_github": { "kept_unchanged": 0, "removed_or_changed": 10 }, "pii_email": { "kept_unchanged": 0, "removed_or_changed": 10 }, "mojibake": { "kept_unchanged": 0, "removed_or_changed": 10 }, "label_swap": { "kept_unchanged": 10, "removed_or_changed": 0 }, "truncated_response": { "kept_unchanged": 10, "removed_or_changed": 0 }, "exact_duplicate": { "kept_unchanged": 10, "removed_or_changed": 0 }, "whitespace_duplicate": { "kept_unchanged": 5, "removed_or_changed": 5 }, "near_duplicate": { "kept_unchanged": 10, "removed_or_changed": 0 }, "eval_exact": { "kept_unchanged": 0, "removed_or_changed": 10 }, "eval_prompt_only": { "kept_unchanged": 0, "removed_or_changed": 10 }, "eval_paraphrase": { "kept_unchanged": 10, "removed_or_changed": 0 } } }, "cookbook_seconds_10k": 0.086, "minhash_seconds_10k": 9.68 }, "results": [ { "file": "sft", "defect": "invalid_json", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 1.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "schema" ], "cookbook_crashed": true, "P_seconds": 1.36 }, { "file": "sft", "defect": "missing_assistant", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 1.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "roles" ], "cookbook_crashed": false, "P_seconds": 1.36 }, { "file": "sft", "defect": "role_order", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "roles" ], "cookbook_crashed": false, "P_seconds": 1.42 }, { "file": "sft", "defect": "empty_response", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 1.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "empty-content" ], "cookbook_crashed": false, "P_seconds": 1.49 }, { "file": "sft", "defect": "whitespace_response", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "empty-content" ], "cookbook_crashed": false, "P_seconds": 1.41 }, { "file": "sft", "defect": "overlong_response", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 0.98, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "context-window" ], "cookbook_crashed": false, "P_seconds": 1.64 }, { "file": "sft", "defect": "overlong_user", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "context-window" ], "cookbook_crashed": false, "P_seconds": 1.67 }, { "file": "sft", "defect": "secret_aws", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "secrets" ], "cookbook_crashed": false, "P_seconds": 1.51 }, { "file": "sft", "defect": "secret_github", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "secrets" ], "cookbook_crashed": false, "P_seconds": 1.44 }, { "file": "sft", "defect": "pii_email", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "pii" ], "cookbook_crashed": false, "P_seconds": 1.57 }, { "file": "sft", "defect": "mojibake", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "encoding" ], "cookbook_crashed": false, "P_seconds": 1.45 }, { "file": "sft", "defect": "label_swap", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 0.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [], "cookbook_crashed": false, "P_seconds": 1.43 }, { "file": "sft", "defect": "truncated_response", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 0.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [], "cookbook_crashed": false, "P_seconds": 1.53 }, { "file": "sft", "defect": "exact_duplicate", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 1.0, "exact_collateral": 0, "minhash_recall": 1.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "duplicates" ], "cookbook_crashed": false, "P_seconds": 1.5 }, { "file": "sft", "defect": "whitespace_duplicate", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 1.0, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [ "duplicates" ], "cookbook_crashed": false, "P_seconds": 1.46 }, { "file": "sft", "defect": "near_duplicate", "n_injected": 100, "rows": 9963, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 0.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.16, "minhash_collateral": 0, "ngram13_recall": 0.0, "ngram13_collateral": 0, "P_rules": [], "cookbook_crashed": false, "P_seconds": 1.6 }, { "file": "sft", "defect": "eval_exact", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 1.0, "ngram13_collateral": 0, "P_rules": [ "eval-overlap" ], "cookbook_crashed": false, "P_seconds": 1.47 }, { "file": "sft", "defect": "eval_prompt_only", "n_injected": 100, "rows": 9963, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.46, "ngram13_collateral": 0, "P_rules": [ "eval-overlap" ], "cookbook_crashed": false, "P_seconds": 1.42 }, { "file": "sft", "defect": "eval_paraphrase", "n_injected": 100, "rows": 9963, "P_error_recall": 0.02, "P_error_collateral": 0, "P_any_recall": 0.02, "P_any_collateral": 0, "cookbook_recall": 0.0, "cookbook_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "ngram13_recall": 0.96, "ngram13_collateral": 0, "P_rules": [ "eval-overlap" ], "cookbook_crashed": false, "P_seconds": 1.42 }, { "file": "pref", "defect": "identical_pair", "n_injected": 100, "rows": 4960, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "P_rules": [ "preference" ] }, { "file": "pref", "defect": "empty_chosen", "n_injected": 100, "rows": 4960, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.01, "minhash_collateral": 0, "P_rules": [ "empty-content" ] }, { "file": "pref", "defect": "flipped_pair", "n_injected": 100, "rows": 4960, "P_error_recall": 0.0, "P_error_collateral": 0, "P_any_recall": 0.0, "P_any_collateral": 0, "exact_recall": 0.0, "exact_collateral": 0, "minhash_recall": 0.0, "minhash_collateral": 0, "P_rules": [] }, { "file": "pref", "defect": "duplicate_pair", "n_injected": 100, "rows": 4960, "P_error_recall": 1.0, "P_error_collateral": 0, "P_any_recall": 1.0, "P_any_collateral": 0, "exact_recall": 1.0, "exact_collateral": 0, "minhash_recall": 1.0, "minhash_collateral": 0, "P_rules": [ "duplicates" ] } ] } }