load hiero format for reading suffix arrays

6 files changed, 116 insertions, 1 deletions

diff --git a/moses/moses.xcodeproj/project.pbxproj b/moses/moses.xcodeproj/project.pbxproj
index 38cb55d45..2393f9e43 100644
--- a/moses/moses.xcodeproj/project.pbxproj
+++ b/moses/moses.xcodeproj/project.pbxproj
@@ -11,6 +11,8 @@
 		1E07292013B3854D004454FD /* AlignmentInfoCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */; };
 		1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */; };
 		1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */; };
+		1E078C21146440A900A707F4 /* RuleTableLoaderHiero.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E078C20146440A900A707F4 /* RuleTableLoaderHiero.h */; };
+		1E078C23146440F700A707F4 /* RuleTableLoaderHiero.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E078C22146440F700A707F4 /* RuleTableLoaderHiero.cpp */; };
 		1E16D0A3144DAA6C00B60B4F /* Base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D087144DAA6C00B60B4F /* Base.cpp */; };
 		1E16D0A4144DAA6C00B60B4F /* Base.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E16D088144DAA6C00B60B4F /* Base.h */; };
 		1E16D0A7144DAA6C00B60B4F /* Factory.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D08B144DAA6C00B60B4F /* Factory.cpp */; };
@@ -279,6 +281,8 @@
 		1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AlignmentInfoCollection.h; path = src/AlignmentInfoCollection.h; sourceTree = "<group>"; };
 		1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryHiero.h; path = src/PhraseDictionaryHiero.h; sourceTree = "<group>"; };
 		1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionaryHiero.cpp; path = src/PhraseDictionaryHiero.cpp; sourceTree = "<group>"; };
+		1E078C20146440A900A707F4 /* RuleTableLoaderHiero.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = RuleTableLoaderHiero.h; path = src/RuleTableLoaderHiero.h; sourceTree = "<group>"; };
+		1E078C22146440F700A707F4 /* RuleTableLoaderHiero.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = RuleTableLoaderHiero.cpp; path = src/RuleTableLoaderHiero.cpp; sourceTree = "<group>"; };
 		1E16D087144DAA6C00B60B4F /* Base.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Base.cpp; path = src/LM/Base.cpp; sourceTree = "<group>"; };
 		1E16D088144DAA6C00B60B4F /* Base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Base.h; path = src/LM/Base.h; sourceTree = "<group>"; };
 		1E16D08B144DAA6C00B60B4F /* Factory.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Factory.cpp; path = src/LM/Factory.cpp; sourceTree = "<group>"; };
@@ -736,6 +740,8 @@
 				1ED0DE221432A0D100C20FBE /* RuleTableLoaderFactory.h */,
 				1ED0DE251432A0D100C20FBE /* RuleTableLoaderStandard.cpp */,
 				1ED0DE261432A0D100C20FBE /* RuleTableLoaderStandard.h */,
+				1E078C20146440A900A707F4 /* RuleTableLoaderHiero.h */,
+				1E078C22146440F700A707F4 /* RuleTableLoaderHiero.cpp */,
 				1ED0FDEB124BB9380029177F /* ScoreComponentCollection.cpp */,
 				1ED0FDEC124BB9380029177F /* ScoreComponentCollection.h */,
 				1ED0FDED124BB9380029177F /* ScoreIndexManager.cpp */,
@@ -1021,6 +1027,7 @@
 				1E3CBD3B145D755D00AF72FC /* WordLattice2.h in Headers */,
 				1E3CBD40145D75F300AF72FC /* WeightedDirectedGraph.h in Headers */,
 				1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */,
+				1E078C21146440A900A707F4 /* RuleTableLoaderHiero.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -1199,6 +1206,7 @@
 				1E3CBD3A145D755D00AF72FC /* WordLattice2.cpp in Sources */,
 				1E3CBD3F145D75F300AF72FC /* WeightedDirectedGraph.cpp in Sources */,
 				1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */,
+				1E078C23146440F700A707F4 /* RuleTableLoaderHiero.cpp in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
diff --git a/moses/src/RuleTableLoaderFactory.cpp b/moses/src/RuleTableLoaderFactory.cpp
index 5287dad3b..7ac97bde2 100644
--- a/moses/src/RuleTableLoaderFactory.cpp
+++ b/moses/src/RuleTableLoaderFactory.cpp
@@ -21,6 +21,7 @@
 
 #include "InputFileStream.h"
 #include "RuleTableLoaderCompact.h"
+#include "RuleTableLoaderHiero.h"
 #include "RuleTableLoaderStandard.h"
 #include "UserMessage.h"
 #include "Util.h"
@@ -49,6 +50,12 @@ std::auto_ptr<RuleTableLoader> RuleTableLoaderFactory::Create(
     UserMessage::Add(msg.str());
     return std::auto_ptr<RuleTableLoader>();
   }
+  else if (tokens[0] == "[X]" && tokens[1] == "|||") {
+    return std::auto_ptr<RuleTableLoader>(new 
+        RuleTableLoaderHiero());
+    
+  }
+  
   return std::auto_ptr<RuleTableLoader>(new RuleTableLoaderStandard());
 }
 
diff --git a/moses/src/RuleTableLoaderHiero.cpp b/moses/src/RuleTableLoaderHiero.cpp
new file mode 100644
index 000000000..6899cfa61
--- /dev/null
+++ b/moses/src/RuleTableLoaderHiero.cpp
@@ -0,0 +1,34 @@
+//
+//  RuleTableLoaderHiero.cpp
+//  moses
+//
+//  Created by Hieu Hoang on 04/11/2011.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#include <iostream>
+#include "RuleTableLoaderHiero.h"
+
+using namespace std;
+
+namespace Moses {
+  
+bool RuleTableLoaderHiero::Load(const std::vector<FactorType> &input,
+          const std::vector<FactorType> &output,
+          std::istream &inStream,
+          const std::vector<float> &weight,
+          size_t tableLimit,
+          const LMList &languageModels,
+          const WordPenaltyProducer* wpProducer,
+          PhraseDictionarySCFG &ruleTable)
+{
+  bool ret = RuleTableLoaderStandard::Load(HieroFormat
+              ,input, output
+              ,inStream, weight
+              ,tableLimit, languageModels
+              ,wpProducer, ruleTable);
+  return ret;
+}
+
+}
+
diff --git a/moses/src/RuleTableLoaderHiero.h b/moses/src/RuleTableLoaderHiero.h
new file mode 100644
index 000000000..a1f8a9205
--- /dev/null
+++ b/moses/src/RuleTableLoaderHiero.h
@@ -0,0 +1,32 @@
+//
+//  RuleTableLoaderHiero.h
+//  moses
+//
+//  Created by Hieu Hoang on 04/11/2011.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#ifndef moses_RuleTableLoaderHiero_h
+#define moses_RuleTableLoaderHiero_h
+
+#include "RuleTableLoaderStandard.h"
+
+namespace Moses {
+
+class RuleTableLoaderHiero : public RuleTableLoaderStandard
+{
+public:
+  bool Load(const std::vector<FactorType> &input,
+            const std::vector<FactorType> &output,
+            std::istream &inStream,
+            const std::vector<float> &weight,
+            size_t tableLimit,
+            const LMList &languageModels,
+            const WordPenaltyProducer* wpProducer,
+            PhraseDictionarySCFG &);
+
+};
+
+}
+
+#endif
diff --git a/moses/src/RuleTableLoaderStandard.cpp b/moses/src/RuleTableLoaderStandard.cpp
index 8b17803e7..f6ac2786f 100644
--- a/moses/src/RuleTableLoaderStandard.cpp
+++ b/moses/src/RuleTableLoaderStandard.cpp
@@ -40,8 +40,26 @@ using namespace std;
 
 namespace Moses
 {
-
 bool RuleTableLoaderStandard::Load(const std::vector<FactorType> &input
+                                   , const std::vector<FactorType> &output
+                                   , std::istream &inStream
+                                   , const std::vector<float> &weight
+                                   , size_t tableLimit
+                                   , const LMList &languageModels
+                                   , const WordPenaltyProducer* wpProducer
+                                   , PhraseDictionarySCFG &ruleTable)
+{
+  bool ret = Load(MosesFormat
+                  ,input, output
+                  ,inStream, weight
+                  ,tableLimit, languageModels
+                  ,wpProducer, ruleTable);
+  return ret;
+
+}
+  
+bool RuleTableLoaderStandard::Load(FormatType format
+                                , const std::vector<FactorType> &input
                                 , const std::vector<FactorType> &output
                                 , std::istream &inStream
                                 , const std::vector<float> &weight
diff --git a/moses/src/RuleTableLoaderStandard.h b/moses/src/RuleTableLoaderStandard.h
index 37c32ea2c..80d00a28b 100644
--- a/moses/src/RuleTableLoaderStandard.h
+++ b/moses/src/RuleTableLoaderStandard.h
@@ -23,9 +23,25 @@
 
 namespace Moses
 {
+  enum FormatType
+  {
+    MosesFormat
+    ,HieroFormat
+  };
 
 class RuleTableLoaderStandard : public RuleTableLoader
 {
+protected:
+
+  bool Load(FormatType format,
+            const std::vector<FactorType> &input,
+            const std::vector<FactorType> &output,
+            std::istream &inStream,
+            const std::vector<float> &weight,
+            size_t tableLimit,
+            const LMList &languageModels,
+            const WordPenaltyProducer* wpProducer,
+            PhraseDictionarySCFG &);
  public:
   bool Load(const std::vector<FactorType> &input,
             const std::vector<FactorType> &output,