diff options
author | phikoehn <pkoehn@inf.ed.ac.uk> | 2012-05-30 03:58:18 +0400 |
---|---|---|
committer | phikoehn <pkoehn@inf.ed.ac.uk> | 2012-05-30 03:58:18 +0400 |
commit | 2e370ed11b0cd8989118891dc4385619837dd39f (patch) | |
tree | e0b8ab423399453a90bc4cacabc323289620042c /scripts/tokenizer | |
parent | fd577d7a65cab923b9102d61873a032654d573a1 (diff) |
more escaping in tokenizer; wrapper for berkeley parser (german)
Diffstat (limited to 'scripts/tokenizer')
-rwxr-xr-x | scripts/tokenizer/deescape-special-chars.perl | 18 | ||||
-rwxr-xr-x | scripts/tokenizer/detokenizer.perl | 18 | ||||
-rwxr-xr-x | scripts/tokenizer/escape-special-chars.perl | 14 | ||||
-rwxr-xr-x | scripts/tokenizer/tokenizer.perl | 14 |
4 files changed, 36 insertions, 28 deletions
diff --git a/scripts/tokenizer/deescape-special-chars.perl b/scripts/tokenizer/deescape-special-chars.perl index 55035ae6d..345555990 100755 --- a/scripts/tokenizer/deescape-special-chars.perl +++ b/scripts/tokenizer/deescape-special-chars.perl @@ -3,13 +3,15 @@ use strict; while(<STDIN>) { - s/\&bar;/\|/g; - s/\</\</g; - s/\>/\>/g; - s/\&bra;/\[/g; - s/\&ket;/\]/g; - s/\[/\[/g; - s/\]/\]/g; - s/\&/\&/g; + s/\&bar;/\|/g; # factor separator + s/\</\</g; # xml + s/\>/\>/g; # xml + s/\&bra;/\[/g; # syntax non-terminal (legacy) + s/\&ket;/\]/g; # syntax non-terminal (legacy) + s/\"/\"/g; # xml + s/\'/\'/g; # xml + s/\[/\[/g; # syntax non-terminal + s/\]/\]/g; # syntax non-terminal + s/\&/\&/g; # escape escape print $_; } diff --git a/scripts/tokenizer/detokenizer.perl b/scripts/tokenizer/detokenizer.perl index e55a1a26e..8233b419c 100755 --- a/scripts/tokenizer/detokenizer.perl +++ b/scripts/tokenizer/detokenizer.perl @@ -66,14 +66,16 @@ sub detokenize { $text = " $text "; $text =~ s/ \@\-\@ /-/g; # de-escape special chars - $text =~ s/\&bar;/\|/g; - $text =~ s/\</\</g; - $text =~ s/\>/\>/g; - $text =~ s/\&bra;/\[/g; - $text =~ s/\&ket;/\]/g; - $text =~ s/\[/\[/g; - $text =~ s/\]/\]/g; - $text =~ s/\&/\&/g; + $text =~ s/\&bar;/\|/g; # factor separator + $text =~ s/\</\</g; # xml + $text =~ s/\>/\>/g; # xml + $text =~ s/\&bra;/\[/g; # syntax non-terminal (legacy) + $text =~ s/\&ket;/\]/g; # syntax non-terminal (legacy) + $text =~ s/\"/\"/g; # xml + $text =~ s/\'/\'/g; # xml + $text =~ s/\[/\[/g; # syntax non-terminal + $text =~ s/\]/\]/g; # syntax non-terminal + $text =~ s/\&/\&/g; # escape escape my $word; my $i; diff --git a/scripts/tokenizer/escape-special-chars.perl b/scripts/tokenizer/escape-special-chars.perl index f4c1b4dd5..5d9690c04 100755 --- a/scripts/tokenizer/escape-special-chars.perl +++ b/scripts/tokenizer/escape-special-chars.perl @@ -12,12 +12,14 @@ while(<STDIN>) { s/ $//g; # special characters in moses - s/\&/\&/g; - s/\|/\&bar;/g; - s/\</\</g; - s/\>/\>/g; - s/\[/\[/g; - s/\]/\]/g; + s/\&/\&/g; # escape escape + s/\|/\&bar;/g; # factor separator + s/\</\</g; # xml + s/\>/\>/g; # xml + s/\'/\'/g; # xml + s/\"/\"/g; # xml + s/\[/\[/g; # syntax non-terminal + s/\]/\]/g; # syntax non-terminal # restore xml instructions s/\<(\S+) translation="([^\"]+)"> (.+?) <\/(\S+)>/\<$1 translation=\"$2\"> $3 <\/$4>/g; diff --git a/scripts/tokenizer/tokenizer.perl b/scripts/tokenizer/tokenizer.perl index 70bb318f7..0cb713740 100755 --- a/scripts/tokenizer/tokenizer.perl +++ b/scripts/tokenizer/tokenizer.perl @@ -149,12 +149,14 @@ sub tokenize { $text =~ s/DOTMULTI/./g; #escape special chars - $text =~ s/\&/\&/g; - $text =~ s/\|/\&bar;/g; - $text =~ s/\</\</g; - $text =~ s/\>/\>/g; - $text =~ s/\[/\[/g; - $text =~ s/\]/\]/g; + $text =~ s/\&/\&/g; # escape escape + $text =~ s/\|/\&bar;/g; # factor separator + $text =~ s/\</\</g; # xml + $text =~ s/\>/\>/g; # xml + $text =~ s/\'/\'/g; # xml + $text =~ s/\"/\"/g; # xml + $text =~ s/\[/\[/g; # syntax non-terminal + $text =~ s/\]/\]/g; # syntax non-terminal #ensure final line break $text .= "\n" unless $text =~ /\n$/; |