bug fix to filter hierarchical

2 files changed, 8 insertions, 0 deletions

diff --git a/scripts/tokenizer/detokenizer.perl b/scripts/tokenizer/detokenizer.perl
index 6f23ab9c6..e2d7ea0bb 100755
--- a/scripts/tokenizer/detokenizer.perl
+++ b/scripts/tokenizer/detokenizer.perl
@@ -68,6 +68,8 @@ sub detokenize {
   $text =~ s/\&bar;/\|/g;
   $text =~ s/\&lt;/\</g;
   $text =~ s/\&gt;/\>/g;
+  $text =~ s/\&bra;/\[/g;
+  $text =~ s/\&ket;/\]/g;
   $text =~ s/\&amp;/\&/g;
 
 	my $word;
diff --git a/scripts/tokenizer/tokenizer.perl b/scripts/tokenizer/tokenizer.perl
index 7158c417a..a97d5e160 100755
--- a/scripts/tokenizer/tokenizer.perl
+++ b/scripts/tokenizer/tokenizer.perl
@@ -70,6 +70,10 @@ sub tokenize {
 	chomp($text);
 	$text = " $text ";
 	
+  # remove ASCII junk
+  $text =~ s/\s+/ /g;
+  $text =~ s/[\000-\037]//g;
+
 	# seperate out all "other" special characters
 	$text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g;
 	
@@ -149,6 +153,8 @@ sub tokenize {
   $text =~ s/\|/\&bar;/g;
   $text =~ s/\</\&lt;/g;
   $text =~ s/\>/\&gt;/g;
+  $text =~ s/\[/\&bra;/g;
+  $text =~ s/\]/\&ket;/g;
 
 	#ensure final line break
 	$text .= "\n" unless $text =~ /\n$/;