all programs in training can take in gzipped file

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4354 1f5c12ca-751b-0410-a591-d2e778427230

4 files changed, 24 insertions, 26 deletions

diff --git a/scripts/training/phrase-extract/Makefile b/scripts/training/phrase-extract/Makefile
index 575f368dd..e47623dd2 100644
--- a/scripts/training/phrase-extract/Makefile
+++ b/scripts/training/phrase-extract/Makefile
@@ -7,11 +7,11 @@ clean:
 .cpp.o:
 	$(CXX) -O6 -g -c $<
 
-extract: tables-core.o SentenceAlignment.o extract.o
-	$(CXX) $^ -o extract
+extract: tables-core.o SentenceAlignment.o extract.o InputFileStream.o
+	$(CXX) $^ -lz -o extract
 
-extract-rules: tables-core.o SentenceAlignment.o SentenceAlignmentWithSyntax.o SyntaxTree.o XmlTree.o HoleCollection.o extract-rules.o ExtractedRule.o
-	$(CXX) $^ -o extract-rules
+extract-rules: tables-core.o SentenceAlignment.o SentenceAlignmentWithSyntax.o SyntaxTree.o XmlTree.o HoleCollection.o extract-rules.o ExtractedRule.o InputFileStream.o
+	$(CXX) $^ -lz -o extract-rules
 
 extract-lex: extract-lex.o InputFileStream.o
 	$(CXX) $^ -lz -o extract-lex
@@ -31,7 +31,7 @@ consolidate-reverse: consolidate-reverse.o tables-core.o InputFileStream.o
 relax-parse: tables-core.o SyntaxTree.o XmlTree.o relax-parse.o
 	$(CXX) $^ -o relax-parse
 
-statistics: tables-core.o AlignmentPhrase.o statistics.o
-	$(CXX) $^ -o statistics
+statistics: tables-core.o AlignmentPhrase.o statistics.o InputFileStream.o
+	$(CXX) $^ -lz -o statistics
 
 
diff --git a/scripts/training/phrase-extract/extract-rules.cpp b/scripts/training/phrase-extract/extract-rules.cpp
index 31dec4cbb..77f7cd01c 100644
--- a/scripts/training/phrase-extract/extract-rules.cpp
+++ b/scripts/training/phrase-extract/extract-rules.cpp
@@ -44,6 +44,7 @@
 #include "SyntaxTree.h"
 #include "tables-core.h"
 #include "XmlTree.h"
+#include "InputFileStream.h"
 
 #define LINE_MAX_LENGTH 500000
 
@@ -226,12 +227,10 @@ int main(int argc, char* argv[])
   cerr << "extracting hierarchical rules" << endl;
 
   // open input files
-  ifstream tFile;
-  ifstream sFile;
-  ifstream aFile;
-  tFile.open(fileNameT);
-  sFile.open(fileNameS);
-  aFile.open(fileNameA);
+  Moses::InputFileStream tFile(fileNameT);
+  Moses::InputFileStream sFile(fileNameS);
+  Moses::InputFileStream aFile(fileNameA);
+
   istream *tFileP = &tFile;
   istream *sFileP = &sFile;
   istream *aFileP = &aFile;
@@ -281,9 +280,9 @@ int main(int argc, char* argv[])
     if (options.onlyOutputSpanInfo) cout << "LOG: PHRASES_END:" << endl; //az: mark end of phrases
   }
 
-  tFile.close();
-  sFile.close();
-  aFile.close();
+  tFile.Close();
+  sFile.Close();
+  aFile.Close();
   // only close if we actually opened it
   if (!options.onlyOutputSpanInfo) {
     extractFile.close();
diff --git a/scripts/training/phrase-extract/extract.cpp b/scripts/training/phrase-extract/extract.cpp
index 0d56de3c3..dee6e2ae3 100644
--- a/scripts/training/phrase-extract/extract.cpp
+++ b/scripts/training/phrase-extract/extract.cpp
@@ -21,6 +21,7 @@
 #include "SafeGetline.h"
 #include "SentenceAlignment.h"
 #include "tables-core.h"
+#include "InputFileStream.h"
 
 using namespace std;
 
@@ -182,12 +183,10 @@ int main(int argc, char* argv[])
   }
 
   // open input files
-  ifstream eFile;
-  ifstream fFile;
-  ifstream aFile;
-  eFile.open(fileNameE);
-  fFile.open(fileNameF);
-  aFile.open(fileNameA);
+  Moses::InputFileStream eFile(fileNameE);
+  Moses::InputFileStream fFile(fileNameF);
+  Moses::InputFileStream aFile(fileNameA);
+
   istream *eFileP = &eFile;
   istream *fFileP = &fFile;
   istream *aFileP = &aFile;
@@ -234,9 +233,9 @@ int main(int argc, char* argv[])
     }
     if (onlyOutputSpanInfo) cout << "LOG: PHRASES_END:" << endl; //az: mark end of phrases
   }
-  eFile.close();
-  fFile.close();
-  aFile.close();
+  eFile.Close();
+  fFile.Close();
+  aFile.Close();
   //az: only close if we actually opened it
   if (!onlyOutputSpanInfo) {
     if (translationFlag) {
diff --git a/scripts/training/phrase-extract/statistics.cpp b/scripts/training/phrase-extract/statistics.cpp
index 075f3db88..1083621ec 100644
--- a/scripts/training/phrase-extract/statistics.cpp
+++ b/scripts/training/phrase-extract/statistics.cpp
@@ -14,6 +14,7 @@
 #include "AlignmentPhrase.h"
 #include "SafeGetline.h"
 #include "tables-core.h"
+#include "InputFileStream.h"
 
 using namespace std;
 
@@ -78,9 +79,8 @@ int main(int argc, char* argv[])
   lexTable.load( fileNameLex );
 
   // sorted phrase extraction file
-  ifstream extractFile;
+  Moses::InputFileStream extractFile(fileNameExtract);
 
-  extractFile.open(fileNameExtract);
   if (extractFile.fail()) {
     cerr << "ERROR: could not open extract file " << fileNameExtract << endl;
     exit(1);