misc/java-utils/ShrinkSentence.java


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

// $Id$

import java.io.*;
import java.util.*;

//used to create language model
public class ShrinkSentence
{
	public static void main(String[] args) throws Exception
	{
		System.err.println("Starting...");

		InputStreamReader inStream = new InputStreamReader(args.length > 0 ? new FileInputStream(args[0]) : System.in
														, "Latin1"); 
		OutputStreamWriter outStream = new OutputStreamWriter(args.length > 1 ? new FileOutputStream(args[1]) : (OutputStream) System.out
														, "Latin1"); 
		
		new ShrinkSentence(inStream, outStream);
		
		System.err.println("End...");
	}

	public ShrinkSentence(Reader inStream, Writer outStream) throws Exception
	{
		BufferedReader inFile = new BufferedReader(inStream); 
		BufferedWriter outFile = new BufferedWriter(outStream); 

		// tokenise
		String inLine;
		int i = 1;
		while ((inLine = inFile.readLine()) != null)
		{
			StringTokenizer st = new StringTokenizer(inLine);
			while (st.hasMoreTokens()) 
		    {
				String word = st.nextToken();
				if (!word.equals("???"))
					outFile.write(word + " ");
		    }
			outFile.write("\n");
			i++;
		}
		outFile.flush();
		outFile.close();
		outFile = null;
		System.err.print("no of lines = " + i);		
	}
}