minor fixes

4 files changed, 14 insertions, 7 deletions

diff --git a/scripts/tokenizer/deescape-special-chars.perl b/scripts/tokenizer/deescape-special-chars.perl
index c98e01ccc..55035ae6d 100755
--- a/scripts/tokenizer/deescape-special-chars.perl
+++ b/scripts/tokenizer/deescape-special-chars.perl
@@ -8,6 +8,8 @@ while(<STDIN>) {
   s/\&gt;/\>/g;
   s/\&bra;/\[/g;
   s/\&ket;/\]/g;
+  s/\&#91;/\[/g;
+  s/\&#93;/\]/g;
   s/\&amp;/\&/g;
   print $_;
 }
diff --git a/scripts/tokenizer/detokenizer.perl b/scripts/tokenizer/detokenizer.perl
index e2d7ea0bb..e55a1a26e 100755
--- a/scripts/tokenizer/detokenizer.perl
+++ b/scripts/tokenizer/detokenizer.perl
@@ -33,8 +33,9 @@ if ($HELP) {
 	exit;
 }
 
-die "No built-in rules for language $language, claim en for default behaviour."
-	if $language !~ /^(cs|en|fr|it)$/;
+if ($language !~ /^(cs|en|fr|it)$/) {
+  print STDERR "Warning: No built-in rules for language $language.\n"
+}
 
 if (!$QUIET) {
 	print STDERR "Detokenizer Version ".'$Revision: 4134 $'."\n";
@@ -70,6 +71,8 @@ sub detokenize {
   $text =~ s/\&gt;/\>/g;
   $text =~ s/\&bra;/\[/g;
   $text =~ s/\&ket;/\]/g;
+  $text =~ s/\&#91;/\[/g;
+  $text =~ s/\&#93;/\]/g;
   $text =~ s/\&amp;/\&/g;
 
 	my $word;
diff --git a/scripts/tokenizer/escape-special-chars.perl b/scripts/tokenizer/escape-special-chars.perl
index 5c4dc9bb3..f4c1b4dd5 100755
--- a/scripts/tokenizer/escape-special-chars.perl
+++ b/scripts/tokenizer/escape-special-chars.perl
@@ -6,18 +6,20 @@ while(<STDIN>) {
   chop;
 
   # avoid general madness
+  s/[\000-\037]//g;
   s/\s+/ /g;
 	s/^ //g;
 	s/ $//g;
-  s/[\000-\037]//g;
 
   # special characters in moses
   s/\&/\&amp;/g;
   s/\|/\&bar;/g;
   s/\</\&lt;/g;
   s/\>/\&gt;/g;
-  s/\[/\&bra;/g;
-  s/\]/\&ket;/g;
+  s/\[/\&#91;/g;
+  s/\]/\&#93;/g;
   
+  # restore xml instructions
+  s/\&lt;(\S+) translation="([^\"]+)"&gt; (.+?) &lt;\/(\S+)&gt;/\<$1 translation=\"$2\"> $3 <\/$4>/g;
   print $_."\n";
 }
diff --git a/scripts/tokenizer/tokenizer.perl b/scripts/tokenizer/tokenizer.perl
index a97d5e160..70bb318f7 100755
--- a/scripts/tokenizer/tokenizer.perl
+++ b/scripts/tokenizer/tokenizer.perl
@@ -153,8 +153,8 @@ sub tokenize {
   $text =~ s/\|/\&bar;/g;
   $text =~ s/\</\&lt;/g;
   $text =~ s/\>/\&gt;/g;
-  $text =~ s/\[/\&bra;/g;
-  $text =~ s/\]/\&ket;/g;
+  $text =~ s/\[/\&#91;/g;
+  $text =~ s/\]/\&#93;/g;
 
 	#ensure final line break
 	$text .= "\n" unless $text =~ /\n$/;