load hiero format for reading suffix arrays

6 files changed, 93 insertions, 0 deletions

diff --git a/moses/moses.xcodeproj/project.pbxproj b/moses/moses.xcodeproj/project.pbxproj
index 700ffde39..38cb55d45 100644
--- a/moses/moses.xcodeproj/project.pbxproj
+++ b/moses/moses.xcodeproj/project.pbxproj
@@ -9,6 +9,8 @@
 /* Begin PBXBuildFile section */
 		1E07291F13B3854D004454FD /* AlignmentInfoCollection.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E07291D13B3854D004454FD /* AlignmentInfoCollection.cpp */; };
 		1E07292013B3854D004454FD /* AlignmentInfoCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */; };
+		1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */; };
+		1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */; };
 		1E16D0A3144DAA6C00B60B4F /* Base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D087144DAA6C00B60B4F /* Base.cpp */; };
 		1E16D0A4144DAA6C00B60B4F /* Base.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E16D088144DAA6C00B60B4F /* Base.h */; };
 		1E16D0A7144DAA6C00B60B4F /* Factory.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D08B144DAA6C00B60B4F /* Factory.cpp */; };
@@ -275,6 +277,8 @@
 /* Begin PBXFileReference section */
 		1E07291D13B3854D004454FD /* AlignmentInfoCollection.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AlignmentInfoCollection.cpp; path = src/AlignmentInfoCollection.cpp; sourceTree = "<group>"; };
 		1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AlignmentInfoCollection.h; path = src/AlignmentInfoCollection.h; sourceTree = "<group>"; };
+		1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryHiero.h; path = src/PhraseDictionaryHiero.h; sourceTree = "<group>"; };
+		1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionaryHiero.cpp; path = src/PhraseDictionaryHiero.cpp; sourceTree = "<group>"; };
 		1E16D087144DAA6C00B60B4F /* Base.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Base.cpp; path = src/LM/Base.cpp; sourceTree = "<group>"; };
 		1E16D088144DAA6C00B60B4F /* Base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Base.h; path = src/LM/Base.h; sourceTree = "<group>"; };
 		1E16D08B144DAA6C00B60B4F /* Factory.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Factory.cpp; path = src/LM/Factory.cpp; sourceTree = "<group>"; };
@@ -709,6 +713,8 @@
 				1ED0FDE0124BB9380029177F /* PhraseDictionaryTree.h */,
 				1ED0FDE1124BB9380029177F /* PhraseDictionaryTreeAdaptor.cpp */,
 				1ED0FDE2124BB9380029177F /* PhraseDictionaryTreeAdaptor.h */,
+				1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */,
+				1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */,
 				1ED0FDE3124BB9380029177F /* PrefixTree.h */,
 				1ED0FDE4124BB9380029177F /* PrefixTreeMap.cpp */,
 				1ED0FDE5124BB9380029177F /* PrefixTreeMap.h */,
@@ -1014,6 +1020,7 @@
 				1E474E12145575CA00178AD5 /* RuleTableLoader.h in Headers */,
 				1E3CBD3B145D755D00AF72FC /* WordLattice2.h in Headers */,
 				1E3CBD40145D75F300AF72FC /* WeightedDirectedGraph.h in Headers */,
+				1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -1191,6 +1198,7 @@
 				1E16D0BD144DAA6C00B60B4F /* SRI.cpp in Sources */,
 				1E3CBD3A145D755D00AF72FC /* WordLattice2.cpp in Sources */,
 				1E3CBD3F145D75F300AF72FC /* WeightedDirectedGraph.cpp in Sources */,
+				1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am
index 8642f92ec..b2994cebc 100644
--- a/moses/src/Makefile.am
+++ b/moses/src/Makefile.am
@@ -231,6 +231,7 @@ libmoses_la_SOURCES = \
         Phrase.cpp \
         PhraseDictionary.cpp \
         PhraseDictionaryDynSuffixArray.cpp \
+        PhraseDictionaryHiero.cpp \
         PhraseDictionaryMemory.cpp \
         PhraseDictionarySCFG.cpp \
         PhraseDictionaryNode.cpp \
diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp
index 589b07f45..bec4b70b6 100644
--- a/moses/src/PhraseDictionary.cpp
+++ b/moses/src/PhraseDictionary.cpp
@@ -24,9 +24,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "PhraseDictionaryTreeAdaptor.h"
 #include "PhraseDictionarySCFG.h"
 #include "PhraseDictionaryOnDisk.h"
+#include "PhraseDictionaryHiero.h"
 #ifndef WIN32
 #include "PhraseDictionaryDynSuffixArray.h"
 #endif
+
 #include "StaticData.h"
 #include "InputType.h"
 #include "TranslationOption.h"
@@ -126,6 +128,24 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst
                          , system->GetWordPenaltyProducer());
     assert(ret);
     return pdm;
+  } else if (m_implementation == Hiero) {
+    // memory phrase table
+    VERBOSE(2,"using Hiero format phrase tables" << std::endl);
+    if (!FileExists(m_filePath) && FileExists(m_filePath + ".gz")) {
+      m_filePath += ".gz";
+      VERBOSE(2,"Using gzipped file" << std::endl);
+    }
+    
+    PhraseDictionaryHiero* pdm  = new PhraseDictionaryHiero(m_numScoreComponent,this);
+    bool ret = pdm->Load(GetInput()
+                         , GetOutput()
+                         , m_filePath
+                         , m_weight
+                         , m_tableLimit
+                         , system->GetLanguageModels()
+                         , system->GetWordPenaltyProducer());
+    assert(ret);
+    return pdm;
   } else if (m_implementation == OnDisk) {
 
     PhraseDictionaryOnDisk* pdta = new PhraseDictionaryOnDisk(m_numScoreComponent, this);
diff --git a/moses/src/PhraseDictionaryHiero.cpp b/moses/src/PhraseDictionaryHiero.cpp
new file mode 100644
index 000000000..3266cabfd
--- /dev/null
+++ b/moses/src/PhraseDictionaryHiero.cpp
@@ -0,0 +1,29 @@
+//
+//  PhraseDictionaryHiero.cpp
+//  moses
+//
+//  Created by Hieu Hoang on 04/11/2011.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#include <iostream>
+#include "PhraseDictionaryHiero.h"
+
+using namespace std;
+
+namespace Moses {
+
+bool PhraseDictionaryHiero::Load(const std::vector<FactorType> &input
+            , const std::vector<FactorType> &output
+            , const std::string &filePath
+            , const std::vector<float> &weight
+            , size_t tableLimit
+            , const LMList &languageModels
+            , const WordPenaltyProducer* wpProducer)
+{
+
+}
+
+}
+
+
diff --git a/moses/src/PhraseDictionaryHiero.h b/moses/src/PhraseDictionaryHiero.h
new file mode 100644
index 000000000..9d7289491
--- /dev/null
+++ b/moses/src/PhraseDictionaryHiero.h
@@ -0,0 +1,34 @@
+//
+//  PhraseDictionaryHiero.h
+//  moses
+//
+//  Created by Hieu Hoang on 04/11/2011.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#ifndef moses_PhraseDictionaryHiero_h
+#define moses_PhraseDictionaryHiero_h
+
+#include "PhraseDictionaryMemory.h"
+
+namespace Moses {
+
+class PhraseDictionaryHiero : public PhraseDictionaryMemory
+{
+public:
+  PhraseDictionaryHiero(size_t numScoreComponent, PhraseDictionaryFeature* feature)
+  : PhraseDictionaryMemory(numScoreComponent,feature) {}
+
+  bool Load(const std::vector<FactorType> &input
+            , const std::vector<FactorType> &output
+            , const std::string &filePath
+            , const std::vector<float> &weight
+            , size_t tableLimit
+            , const LMList &languageModels
+            , const WordPenaltyProducer* wpProducer);
+
+};
+
+}
+
+#endif
diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h
index a40c79baa..bc1707886 100644
--- a/moses/src/TypeDef.h
+++ b/moses/src/TypeDef.h
@@ -174,6 +174,7 @@ enum PhraseTableImplementation {
   ,SCFG					= 6
   //,BerkeleyDb	= 7
   ,SuffixArray	= 8
+  ,Hiero        = 9
 };
 
 enum InputTypeEnum {