Welcome to mirror list, hosted at ThFree Co, Russian Federation.

make-factor-stem.perl « wrappers « training « scripts - github.com/moses-smt/mosesdecoder.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
blob: 60aca0b3462afee3dda4ee32a965e9b5e8d3970b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/usr/bin/env perl 

use warnings;
use strict;

my ($size,$in,$out) = @ARGV;

open(IN,$in);
open(OUT,">$out");
binmode(IN, ":utf8");
binmode(OUT, ":utf8");

while(<IN>) {
    my $first = 1;
    chomp; s/\s+/ /g; s/^ //; s/ $//;
    foreach my $word (split) {
        if (length($word) > $size) {
	    $word = substr($word,0,$size);
        }
	print OUT " " unless $first; $first = 0;
	print OUT $word;
    }
    print OUT "\n";
}
close(OUT);
close(IN);