From 3bd33fbf685c6b16747d6ec4c1026d4fa6d04966 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Sun, 2 Aug 2026 15:31:53 +0100 Subject: Fix in-word metric: strict (both-sides, 58% random / 0% clean) vs touches (95%); blog/results caveat corrected to the strict discriminator --- src/synthetic_pair.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/synthetic_pair.py b/src/synthetic_pair.py index 4c64379..bfe5b8b 100644 --- a/src/synthetic_pair.py +++ b/src/synthetic_pair.py @@ -72,8 +72,11 @@ def prep(clean=False): noise_pos = sorted(rng.choice(avail, size=n_target, replace=False).tolist()) # sanity: fraction of noise insertions that slice inside a word - in_word = sum(1 for p in noise_pos - if 0 < p < len(text) and (text[p - 1].isalnum() and text[p].isalnum())) + # strict: letter on BOTH sides (th#e, ki#ng); touches: letter on either side + strict = sum(1 for p in noise_pos + if 0 < p < len(text) and (text[p - 1].isalnum() and text[p].isalnum())) + touches = sum(1 for p in noise_pos + if 0 < p < len(text) and (text[p - 1].isalnum() or text[p].isalnum())) # insert with offset (both sets sorted -> single merge pass) insertions = [(p, T_STRUCT) for p in struct_pos] + [(p, T_NOISE) for p in noise_pos] @@ -91,7 +94,8 @@ def prep(clean=False): print(f"prep({mode}): '{T_STRUCT}' x{n_target} after '{TRIGGER.strip()}', " f"'{T_NOISE}' x{n_target} {mode}, " f"freq each = {n_target/len(modified):.4%}, " - f"in-word '#' = {in_word}/{n_target} ({in_word/n_target:.1%})") + f"in-word '#' = {strict}/{n_target} ({strict/n_target:.1%}), " + f"touches word = {touches}/{n_target} ({touches/n_target:.1%})") # build vocab (existing chars + the two synthetic) chars = sorted(set(text)) + [T_STRUCT, T_NOISE] -- cgit v1.2.3