diff options
author | Hieu Hoang <hieuhoang@Hieus-MacBook.local> | 2011-11-04 19:43:42 +0400 |
---|---|---|
committer | Hieu Hoang <hieuhoang@Hieus-MacBook.local> | 2011-11-04 19:43:42 +0400 |
commit | 9b9ca34302e6be2c2799bfb6e430d12aa6f8dc38 (patch) | |
tree | a0d6303730247c26cfddc097d3c411457aa9d466 /moses | |
parent | df949b87deecb95738d7edd2f23c2afc2afc68b9 (diff) |
load hiero format for reading suffix arrays
Diffstat (limited to 'moses')
-rw-r--r-- | moses/moses.xcodeproj/project.pbxproj | 8 | ||||
-rw-r--r-- | moses/src/Makefile.am | 1 | ||||
-rw-r--r-- | moses/src/PhraseDictionary.cpp | 20 | ||||
-rw-r--r-- | moses/src/PhraseDictionaryHiero.cpp | 29 | ||||
-rw-r--r-- | moses/src/PhraseDictionaryHiero.h | 34 | ||||
-rw-r--r-- | moses/src/TypeDef.h | 1 |
6 files changed, 93 insertions, 0 deletions
diff --git a/moses/moses.xcodeproj/project.pbxproj b/moses/moses.xcodeproj/project.pbxproj index 700ffde39..38cb55d45 100644 --- a/moses/moses.xcodeproj/project.pbxproj +++ b/moses/moses.xcodeproj/project.pbxproj @@ -9,6 +9,8 @@ /* Begin PBXBuildFile section */ 1E07291F13B3854D004454FD /* AlignmentInfoCollection.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E07291D13B3854D004454FD /* AlignmentInfoCollection.cpp */; }; 1E07292013B3854D004454FD /* AlignmentInfoCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */; }; + 1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */; }; + 1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */; }; 1E16D0A3144DAA6C00B60B4F /* Base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D087144DAA6C00B60B4F /* Base.cpp */; }; 1E16D0A4144DAA6C00B60B4F /* Base.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E16D088144DAA6C00B60B4F /* Base.h */; }; 1E16D0A7144DAA6C00B60B4F /* Factory.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E16D08B144DAA6C00B60B4F /* Factory.cpp */; }; @@ -275,6 +277,8 @@ /* Begin PBXFileReference section */ 1E07291D13B3854D004454FD /* AlignmentInfoCollection.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AlignmentInfoCollection.cpp; path = src/AlignmentInfoCollection.cpp; sourceTree = "<group>"; }; 1E07291E13B3854D004454FD /* AlignmentInfoCollection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AlignmentInfoCollection.h; path = src/AlignmentInfoCollection.h; sourceTree = "<group>"; }; + 1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryHiero.h; path = src/PhraseDictionaryHiero.h; sourceTree = "<group>"; }; + 1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionaryHiero.cpp; path = src/PhraseDictionaryHiero.cpp; sourceTree = "<group>"; }; 1E16D087144DAA6C00B60B4F /* Base.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Base.cpp; path = src/LM/Base.cpp; sourceTree = "<group>"; }; 1E16D088144DAA6C00B60B4F /* Base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Base.h; path = src/LM/Base.h; sourceTree = "<group>"; }; 1E16D08B144DAA6C00B60B4F /* Factory.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Factory.cpp; path = src/LM/Factory.cpp; sourceTree = "<group>"; }; @@ -709,6 +713,8 @@ 1ED0FDE0124BB9380029177F /* PhraseDictionaryTree.h */, 1ED0FDE1124BB9380029177F /* PhraseDictionaryTreeAdaptor.cpp */, 1ED0FDE2124BB9380029177F /* PhraseDictionaryTreeAdaptor.h */, + 1E078C1E14643C2000A707F4 /* PhraseDictionaryHiero.cpp */, + 1E078C1B14643AED00A707F4 /* PhraseDictionaryHiero.h */, 1ED0FDE3124BB9380029177F /* PrefixTree.h */, 1ED0FDE4124BB9380029177F /* PrefixTreeMap.cpp */, 1ED0FDE5124BB9380029177F /* PrefixTreeMap.h */, @@ -1014,6 +1020,7 @@ 1E474E12145575CA00178AD5 /* RuleTableLoader.h in Headers */, 1E3CBD3B145D755D00AF72FC /* WordLattice2.h in Headers */, 1E3CBD40145D75F300AF72FC /* WeightedDirectedGraph.h in Headers */, + 1E078C1C14643AED00A707F4 /* PhraseDictionaryHiero.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1191,6 +1198,7 @@ 1E16D0BD144DAA6C00B60B4F /* SRI.cpp in Sources */, 1E3CBD3A145D755D00AF72FC /* WordLattice2.cpp in Sources */, 1E3CBD3F145D75F300AF72FC /* WeightedDirectedGraph.cpp in Sources */, + 1E078C1F14643C2000A707F4 /* PhraseDictionaryHiero.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am index 8642f92ec..b2994cebc 100644 --- a/moses/src/Makefile.am +++ b/moses/src/Makefile.am @@ -231,6 +231,7 @@ libmoses_la_SOURCES = \ Phrase.cpp \ PhraseDictionary.cpp \ PhraseDictionaryDynSuffixArray.cpp \ + PhraseDictionaryHiero.cpp \ PhraseDictionaryMemory.cpp \ PhraseDictionarySCFG.cpp \ PhraseDictionaryNode.cpp \ diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp index 589b07f45..bec4b70b6 100644 --- a/moses/src/PhraseDictionary.cpp +++ b/moses/src/PhraseDictionary.cpp @@ -24,9 +24,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "PhraseDictionaryTreeAdaptor.h" #include "PhraseDictionarySCFG.h" #include "PhraseDictionaryOnDisk.h" +#include "PhraseDictionaryHiero.h" #ifndef WIN32 #include "PhraseDictionaryDynSuffixArray.h" #endif + #include "StaticData.h" #include "InputType.h" #include "TranslationOption.h" @@ -126,6 +128,24 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst , system->GetWordPenaltyProducer()); assert(ret); return pdm; + } else if (m_implementation == Hiero) { + // memory phrase table + VERBOSE(2,"using Hiero format phrase tables" << std::endl); + if (!FileExists(m_filePath) && FileExists(m_filePath + ".gz")) { + m_filePath += ".gz"; + VERBOSE(2,"Using gzipped file" << std::endl); + } + + PhraseDictionaryHiero* pdm = new PhraseDictionaryHiero(m_numScoreComponent,this); + bool ret = pdm->Load(GetInput() + , GetOutput() + , m_filePath + , m_weight + , m_tableLimit + , system->GetLanguageModels() + , system->GetWordPenaltyProducer()); + assert(ret); + return pdm; } else if (m_implementation == OnDisk) { PhraseDictionaryOnDisk* pdta = new PhraseDictionaryOnDisk(m_numScoreComponent, this); diff --git a/moses/src/PhraseDictionaryHiero.cpp b/moses/src/PhraseDictionaryHiero.cpp new file mode 100644 index 000000000..3266cabfd --- /dev/null +++ b/moses/src/PhraseDictionaryHiero.cpp @@ -0,0 +1,29 @@ +// +// PhraseDictionaryHiero.cpp +// moses +// +// Created by Hieu Hoang on 04/11/2011. +// Copyright 2011 __MyCompanyName__. All rights reserved. +// + +#include <iostream> +#include "PhraseDictionaryHiero.h" + +using namespace std; + +namespace Moses { + +bool PhraseDictionaryHiero::Load(const std::vector<FactorType> &input + , const std::vector<FactorType> &output + , const std::string &filePath + , const std::vector<float> &weight + , size_t tableLimit + , const LMList &languageModels + , const WordPenaltyProducer* wpProducer) +{ + +} + +} + + diff --git a/moses/src/PhraseDictionaryHiero.h b/moses/src/PhraseDictionaryHiero.h new file mode 100644 index 000000000..9d7289491 --- /dev/null +++ b/moses/src/PhraseDictionaryHiero.h @@ -0,0 +1,34 @@ +// +// PhraseDictionaryHiero.h +// moses +// +// Created by Hieu Hoang on 04/11/2011. +// Copyright 2011 __MyCompanyName__. All rights reserved. +// + +#ifndef moses_PhraseDictionaryHiero_h +#define moses_PhraseDictionaryHiero_h + +#include "PhraseDictionaryMemory.h" + +namespace Moses { + +class PhraseDictionaryHiero : public PhraseDictionaryMemory +{ +public: + PhraseDictionaryHiero(size_t numScoreComponent, PhraseDictionaryFeature* feature) + : PhraseDictionaryMemory(numScoreComponent,feature) {} + + bool Load(const std::vector<FactorType> &input + , const std::vector<FactorType> &output + , const std::string &filePath + , const std::vector<float> &weight + , size_t tableLimit + , const LMList &languageModels + , const WordPenaltyProducer* wpProducer); + +}; + +} + +#endif diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index a40c79baa..bc1707886 100644 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h @@ -174,6 +174,7 @@ enum PhraseTableImplementation { ,SCFG = 6 //,BerkeleyDb = 7 ,SuffixArray = 8 + ,Hiero = 9 }; enum InputTypeEnum { |