diff options
Diffstat (limited to 'src/synthetic_pair.py')
| -rw-r--r-- | src/synthetic_pair.py | 10 |
1 files changed, 7 insertions, 3 deletions
diff --git a/src/synthetic_pair.py b/src/synthetic_pair.py index 4c64379..bfe5b8b 100644 --- a/src/synthetic_pair.py +++ b/src/synthetic_pair.py @@ -72,8 +72,11 @@ def prep(clean=False): noise_pos = sorted(rng.choice(avail, size=n_target, replace=False).tolist()) # sanity: fraction of noise insertions that slice inside a word - in_word = sum(1 for p in noise_pos - if 0 < p < len(text) and (text[p - 1].isalnum() and text[p].isalnum())) + # strict: letter on BOTH sides (th#e, ki#ng); touches: letter on either side + strict = sum(1 for p in noise_pos + if 0 < p < len(text) and (text[p - 1].isalnum() and text[p].isalnum())) + touches = sum(1 for p in noise_pos + if 0 < p < len(text) and (text[p - 1].isalnum() or text[p].isalnum())) # insert with offset (both sets sorted -> single merge pass) insertions = [(p, T_STRUCT) for p in struct_pos] + [(p, T_NOISE) for p in noise_pos] @@ -91,7 +94,8 @@ def prep(clean=False): print(f"prep({mode}): '{T_STRUCT}' x{n_target} after '{TRIGGER.strip()}', " f"'{T_NOISE}' x{n_target} {mode}, " f"freq each = {n_target/len(modified):.4%}, " - f"in-word '#' = {in_word}/{n_target} ({in_word/n_target:.1%})") + f"in-word '#' = {strict}/{n_target} ({strict/n_target:.1%}), " + f"touches word = {touches}/{n_target} ({touches/n_target:.1%})") # build vocab (existing chars + the two synthetic) chars = sorted(set(text)) + [T_STRUCT, T_NOISE] |