diff options
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/recaser/train-truecaser.perl | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/scripts/recaser/train-truecaser.perl b/scripts/recaser/train-truecaser.perl index 4f600a640..94ddbf2fa 100755 --- a/scripts/recaser/train-truecaser.perl +++ b/scripts/recaser/train-truecaser.perl @@ -44,6 +44,12 @@ while(<CORPUS>) { $firstWordOfSentence = 1; } + if ($currentWord !~ /[\p{Ll}\p{Lu}\p{Lt}]/) { + # skip words with nothing to case + $firstWordOfSentence = 0; + next; + } + my $currentWordWeight = 0; if (! $firstWordOfSentence) { $currentWordWeight = 1; |