lotsa minor changes: mostly bug fixes, tokenizer now esacapes special Moses characters (|<>&)

2 files changed, 17 insertions, 4 deletions

diff --git a/scripts/tokenizer/detokenizer.perl b/scripts/tokenizer/detokenizer.perl
index d8d6e5da7..6f23ab9c6 100755
--- a/scripts/tokenizer/detokenizer.perl
+++ b/scripts/tokenizer/detokenizer.perl
@@ -63,8 +63,13 @@ sub detokenize {
 	my($text) = @_;
 	chomp($text);
 	$text = " $text ";
-        $text =~ s/ \@\-\@ /-/g;
-	
+  $text =~ s/ \@\-\@ /-/g;
+  # de-escape special chars
+  $text =~ s/\&bar;/\|/g;
+  $text =~ s/\&lt;/\</g;
+  $text =~ s/\&gt;/\>/g;
+  $text =~ s/\&amp;/\&/g;
+
 	my $word;
 	my $i;
 	my @words = split(/ /,$text);
diff --git a/scripts/tokenizer/tokenizer.perl b/scripts/tokenizer/tokenizer.perl
index 84fdc3462..7158c417a 100755
--- a/scripts/tokenizer/tokenizer.perl
+++ b/scripts/tokenizer/tokenizer.perl
@@ -18,6 +18,7 @@ my $language = "en";
 my $QUIET = 0;
 my $HELP = 0;
 my $AGGRESSIVE = 0;
+my $SKIP_XML = 0;
 
 #my $start = [ Time::HiRes::gettimeofday( ) ];
 
@@ -27,6 +28,7 @@ while (@ARGV) {
 	/^-l$/ && ($language = shift, next);
 	/^-q$/ && ($QUIET = 1, next);
 	/^-h$/ && ($HELP = 1, next);
+	/^-x$/ && ($SKIP_XML = 1, next);
 	/^-a$/ && ($AGGRESSIVE = 1, next);
 }
 
@@ -50,7 +52,7 @@ if (scalar(%NONBREAKING_PREFIX) eq 0){
 }
 
 while(<STDIN>) {
-	if (/^<.+>$/ || /^\s*$/) {
+	if (($SKIP_XML && /^<.+>$/) || /^\s*$/) {
 		#don't try to tokenize XML/HTML tag lines
 		print $_;
 	}
@@ -141,7 +143,13 @@ sub tokenize {
 		$text =~ s/DOTDOTMULTI/DOTMULTI./g;
 	}
 	$text =~ s/DOTMULTI/./g;
-	
+
+  #escape special chars
+  $text =~ s/\&/\&amp;/g;
+  $text =~ s/\|/\&bar;/g;
+  $text =~ s/\</\&lt;/g;
+  $text =~ s/\>/\&gt;/g;
+
 	#ensure final line break
 	$text .= "\n" unless $text =~ /\n$/;