diff options
| author | Flammie Pirinen <flammie@iki.fi> | 2015-04-07 18:19:07 +0300 |
|---|---|---|
| committer | Flammie Pirinen <flammie@iki.fi> | 2015-04-07 18:19:07 +0300 |
| commit | fc8ee03b8d51e7c2533dd685042b75c32528d380 (patch) | |
| tree | b2c6049b831147b8d3801ae7fbf0906e908c13ce /scripts | |
| parent | ef52bc66f604a59c1c637d1c59b277df0bfbbbaa (diff) | |
examples
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/tokenizer/detokenizer.perl | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/scripts/tokenizer/detokenizer.perl b/scripts/tokenizer/detokenizer.perl index 908158da6..27e315840 100755 --- a/scripts/tokenizer/detokenizer.perl +++ b/scripts/tokenizer/detokenizer.perl @@ -178,6 +178,7 @@ sub detokenize { } elsif (($language eq "fi") && ($words[$i-1] =~ /:$/) && ($words[$i] =~ /^(N|n|A|a|Ä|ä|ssa|Ssa|ssä|Ssä|sta|stä|Sta|Stä|hun|Hun|hyn|Hyn|han|Han|hän|Hän|hön|Hön|un|Un|yn|Yn|an|An|än|Än|ön|Ön|seen|Seen|lla|Lla|llä|Llä|lta|Lta|ltä|Ltä|lle|Lle|ksi|Ksi|kse|Kse|tta|Tta|ine|Ine)(ni|si|mme|nne|nsa)?(ko|kö|han|hän|pa|pä|kaan|kään|kin)?$/)) { # Finnish : without intervening space if followed by case suffix + # EU:N EU:n EU:ssa EU:sta EU:hun EU:iin ... $text=$text. lc $words[$i]; $prependSpace = " "; } else { |