diff options
author | Ulrich Germann <ulrich.germann@gmail.com> | 2015-04-05 17:51:50 +0300 |
---|---|---|
committer | Ulrich Germann <ulrich.germann@gmail.com> | 2015-04-05 17:51:50 +0300 |
commit | 3e2f878576880e4ae33d3f8e8d63ae989e76fc47 (patch) | |
tree | 2bee6e9e82b3bb3fc6926b2fef98497447d4d776 /moses | |
parent | 34776c5e7cf0806d3a8b1d5725d87a9d573d4635 (diff) | |
parent | 02185a85fb5f78cf99228e034dc62f02c488dc8f (diff) |
Merge branch 'master' into mmt-dev
Conflicts:
Jamroot
moses/TranslationModel/UG/mmsapt.h
Diffstat (limited to 'moses')
42 files changed, 62 insertions, 491 deletions
diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index d221bd0fc..4856b3656 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -19,7 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ -#include <stdio.h> +#include <cstdio> #include "ChartManager.h" #include "ChartCell.h" #include "ChartHypothesis.h" diff --git a/moses/FF/ExternalFeature.cpp b/moses/FF/ExternalFeature.cpp deleted file mode 100644 index 10800d24d..000000000 --- a/moses/FF/ExternalFeature.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "ExternalFeature.h" -#include <dlfcn.h> -#include <stdlib.h> -#include <iostream> -#include "util/exception.hh" - -using namespace std; - -namespace Moses -{ -ExternalFeatureState::ExternalFeatureState(int stateSize, void *data) -{ - m_stateSize = stateSize; - m_data = malloc(stateSize); - memcpy(m_data, data, stateSize); -} - -void ExternalFeature::Load() -{ - string nparam = "testing"; - - if (m_path.size() < 1) { - UTIL_THROW2("External requires a path to a dynamic library"); - } - lib_handle = dlopen(m_path.c_str(), RTLD_LAZY); - if (!lib_handle) { - UTIL_THROW2("dlopen reports: " << dlerror() << ". Did you provide a full path to the dynamic library?";); - } - CdecFF* (*fn)(const string&) = - (CdecFF* (*)(const string&))(dlsym(lib_handle, "create_ff")); - if (!fn) { - UTIL_THROW2("dlsym reports: " << dlerror()); - } - ff_ext = (*fn)(nparam); - m_stateSize = ff_ext->StateSize(); - -} - -ExternalFeature::~ExternalFeature() -{ - delete ff_ext; - dlclose(lib_handle); -} - -void ExternalFeature::SetParameter(const std::string& key, const std::string& value) -{ - if (key == "path") { - m_path = value; - } else { - StatefulFeatureFunction::SetParameter(key, value); - } -} - -FFState* ExternalFeature::EvaluateWhenApplied( - const Hypothesis& cur_hypo, - const FFState* prev_state, - ScoreComponentCollection* accumulator) const -{ - return new ExternalFeatureState(m_stateSize); -} - -FFState* ExternalFeature::EvaluateWhenApplied( - const ChartHypothesis& /* cur_hypo */, - int /* featureID - used to index the state in the previous hypotheses */, - ScoreComponentCollection* accumulator) const -{ - return new ExternalFeatureState(m_stateSize); -} - - -} - diff --git a/moses/FF/ExternalFeature.h b/moses/FF/ExternalFeature.h deleted file mode 100644 index 6c0fb829e..000000000 --- a/moses/FF/ExternalFeature.h +++ /dev/null @@ -1,100 +0,0 @@ -#pragma once - -#include <string> -#include <cstring> -#include <cstdlib> -#include "StatefulFeatureFunction.h" -#include "FFState.h" - -namespace Moses -{ -class CdecFF; - -class ExternalFeatureState : public FFState -{ -protected: - int m_stateSize; - void *m_data; -public: - ExternalFeatureState(int stateSize) - :m_stateSize(stateSize) - ,m_data(NULL) { - } - ExternalFeatureState(int stateSize, void *data); - - ~ExternalFeatureState() { - free(m_data); - } - - int Compare(const FFState& other) const { - const ExternalFeatureState &otherFF = static_cast<const ExternalFeatureState&>(other); - int ret = memcmp(m_data, otherFF.m_data, m_stateSize); - return ret; - } -}; - -// copied from cdec -class ExternalFeature : public StatefulFeatureFunction -{ -public: - ExternalFeature(const std::string &line) - :StatefulFeatureFunction(line) { - ReadParameters(); - } - ~ExternalFeature(); - - void Load(); - - bool IsUseable(const FactorMask &mask) const { - return true; - } - - void SetParameter(const std::string& key, const std::string& value); - - void EvaluateInIsolation(const Phrase &source - , const TargetPhrase &targetPhrase - , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection &estimatedFutureScore) const { - } - void EvaluateWithSourceContext(const InputType &input - , const InputPath &inputPath - , const TargetPhrase &targetPhrase - , const StackVec *stackVec - , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection *estimatedFutureScore = NULL) const { - } - - void EvaluateTranslationOptionListWithSourceContext(const InputType &input - , const TranslationOptionList &translationOptionList) const { - } - - FFState* EvaluateWhenApplied( - const Hypothesis& cur_hypo, - const FFState* prev_state, - ScoreComponentCollection* accumulator) const; - - FFState* EvaluateWhenApplied( - const ChartHypothesis& /* cur_hypo */, - int /* featureID - used to index the state in the previous hypotheses */, - ScoreComponentCollection* accumulator) const; - - virtual const FFState* EmptyHypothesisState(const InputType &input) const { - return new ExternalFeatureState(m_stateSize); - } - -protected: - std::string m_path; - void* lib_handle; - CdecFF *ff_ext; - int m_stateSize; -}; - -class CdecFF -{ -public: - virtual ~CdecFF() {} - virtual int StateSize() const = 0; -}; - -} - diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index 218e458ff..80e4a4243 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -37,7 +37,6 @@ #include "moses/FF/PhrasePenalty.h" #include "moses/FF/OSM-Feature/OpSequenceModel.h" #include "moses/FF/ControlRecombination.h" -#include "moses/FF/ExternalFeature.h" #include "moses/FF/ConstrainedDecoding.h" #include "moses/FF/SoftSourceSyntacticConstraintsFeature.h" #include "moses/FF/CoveredReferenceFeature.h" @@ -229,7 +228,6 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(ControlRecombination); MOSES_FNAME(ConstrainedDecoding); MOSES_FNAME(CoveredReferenceFeature); - MOSES_FNAME(ExternalFeature); MOSES_FNAME(SourceGHKMTreeInputMatchFeature); MOSES_FNAME(SoftSourceSyntacticConstraintsFeature); MOSES_FNAME(TreeStructureFeature); diff --git a/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp b/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp index 05a3c92e0..d57c42d99 100644 --- a/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp +++ b/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp @@ -1,6 +1,6 @@ #include <vector> #include <limits> -#include <assert.h> +#include <cassert> #include "SoftSourceSyntacticConstraintsFeature.h" #include "moses/StaticData.h" #include "moses/InputFileStream.h" diff --git a/moses/FF/SourceGHKMTreeInputMatchFeature.cpp b/moses/FF/SourceGHKMTreeInputMatchFeature.cpp index 15509f1e4..d33cbf94a 100644 --- a/moses/FF/SourceGHKMTreeInputMatchFeature.cpp +++ b/moses/FF/SourceGHKMTreeInputMatchFeature.cpp @@ -1,6 +1,6 @@ #include <map> #include <vector> -#include <assert.h> +#include <cassert> #include "SourceGHKMTreeInputMatchFeature.h" #include "moses/StaticData.h" #include "moses/InputFileStream.h" diff --git a/moses/FF/VW/VWFeatureBase.h b/moses/FF/VW/VWFeatureBase.h index 04eb6974a..d0f5ebebe 100644 --- a/moses/FF/VW/VWFeatureBase.h +++ b/moses/FF/VW/VWFeatureBase.h @@ -3,7 +3,7 @@ #include <string> #include <boost/thread/tss.hpp> -#include "Classifier.h" +#include "vw/Classifier.h" #include "moses/TypeDef.h" #include "moses/Util.h" #include "moses/FF/StatelessFeatureFunction.h" diff --git a/moses/FF/WordTranslationFeature.cpp b/moses/FF/WordTranslationFeature.cpp index 4c0233885..05883df12 100644 --- a/moses/FF/WordTranslationFeature.cpp +++ b/moses/FF/WordTranslationFeature.cpp @@ -22,8 +22,8 @@ WordTranslationFeature::WordTranslationFeature(const std::string &line) ,m_simple(true) ,m_sourceContext(false) ,m_targetContext(false) - ,m_ignorePunctuation(false) ,m_domainTrigger(false) + ,m_ignorePunctuation(false) { VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ..."); ReadParameters(); diff --git a/moses/Hypothesis.cpp b/moses/Hypothesis.cpp index 3707aed18..79a469523 100644 --- a/moses/Hypothesis.cpp +++ b/moses/Hypothesis.cpp @@ -484,9 +484,7 @@ namespace Moses targetOffset += tp.GetSize(); } - // Removing std::endl here breaks -alignment-output-file, so stop doing that, please :) - // Or fix it somewhere else. - out << std::endl; + // Used by --print-alignment-info, so no endl } void diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index 499886505..e86275050 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <cstring> #include <iostream> #include <memory> -#include <stdlib.h> +#include <cstdlib> #include <boost/shared_ptr.hpp> #include <boost/lexical_cast.hpp> @@ -169,9 +169,9 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageM :LanguageModel(copy_from.GetArgLine()), m_ngram(copy_from.m_ngram), // TODO: don't copy this. - m_lmIdLookup(copy_from.m_lmIdLookup), + m_beginSentenceFactor(copy_from.m_beginSentenceFactor), m_factorType(copy_from.m_factorType), - m_beginSentenceFactor(copy_from.m_beginSentenceFactor) + m_lmIdLookup(copy_from.m_lmIdLookup) { } diff --git a/moses/LM/Remote.cpp b/moses/LM/Remote.cpp index 4527a52a7..af02a6617 100644 --- a/moses/LM/Remote.cpp +++ b/moses/LM/Remote.cpp @@ -1,5 +1,5 @@ -#include <stdio.h> -#include <stdlib.h> +#include <cstdio> +#include <cstdlib> #include <unistd.h> #include <sys/types.h> #include <sys/socket.h> diff --git a/moses/Manager.cpp b/moses/Manager.cpp index e368c58f7..cb91a9d29 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -1876,8 +1876,7 @@ void Manager::OutputAlignment(ostream &out, const vector<const Hypothesis *> &ed targetOffset += tp.GetSize(); } - // Removing std::endl here breaks -alignment-output-file, so stop doing that, please :) - // Or fix it somewhere else. + // Used by --alignment-output-file so requires endl out << std::endl; } @@ -2032,6 +2031,8 @@ void Manager::OutputBestHypo(const Moses::TrellisPath &path, long /*translationI void Manager::OutputAlignment(std::ostringstream &out, const TrellisPath &path) const { Hypothesis::OutputAlignment(out, path.GetEdges()); + // Used by --alignment-output-file so requires endl + out << std::endl; } } // namespace diff --git a/moses/PDTAimp.cpp b/moses/PDTAimp.cpp index 1b9e789ce..6770a5c17 100644 --- a/moses/PDTAimp.cpp +++ b/moses/PDTAimp.cpp @@ -222,7 +222,6 @@ void PDTAimp::CacheSource(ConfusionNet const& src) TRACE_ERR("\n"); } - typedef StringTgtCand::Tokens sPhrase; typedef std::map<StringTgtCand::Tokens,TScores> E2Costs; std::map<Range,E2Costs> cov2cand; diff --git a/moses/PP/CountsPhraseProperty.cpp b/moses/PP/CountsPhraseProperty.cpp index 00bc08011..a8cefe37f 100644 --- a/moses/PP/CountsPhraseProperty.cpp +++ b/moses/PP/CountsPhraseProperty.cpp @@ -1,6 +1,6 @@ #include "moses/PP/CountsPhraseProperty.h" #include <sstream> -#include <assert.h> +#include <cassert> namespace Moses { diff --git a/moses/PP/NonTermContextProperty.cpp b/moses/PP/NonTermContextProperty.cpp index d1ea6a554..3126e90d9 100644 --- a/moses/PP/NonTermContextProperty.cpp +++ b/moses/PP/NonTermContextProperty.cpp @@ -1,6 +1,6 @@ #include "moses/PP/NonTermContextProperty.h" #include <string> -#include <assert.h> +#include <cassert> #include "moses/Util.h" #include "moses/FactorCollection.h" diff --git a/moses/PP/SourceLabelsPhraseProperty.cpp b/moses/PP/SourceLabelsPhraseProperty.cpp index efe5ae741..18229fd1a 100644 --- a/moses/PP/SourceLabelsPhraseProperty.cpp +++ b/moses/PP/SourceLabelsPhraseProperty.cpp @@ -5,7 +5,7 @@ #include <sstream> #include <string> #include <queue> -#include <assert.h> +#include <cassert> #include <limits> namespace Moses diff --git a/moses/SentenceStats.h b/moses/SentenceStats.h index c1650eac1..998adc74f 100644 --- a/moses/SentenceStats.h +++ b/moses/SentenceStats.h @@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <iostream> #include <string> #include <vector> -#include <time.h> +#include <ctime> #include "Timer.h" #include "Phrase.h" #include "Hypothesis.h" diff --git a/moses/Syntax/Cube.cpp b/moses/Syntax/Cube.cpp index 0249291d3..6cbf72903 100644 --- a/moses/Syntax/Cube.cpp +++ b/moses/Syntax/Cube.cpp @@ -56,7 +56,7 @@ void Cube::CreateNeighbours(const std::vector<int> &coordinates) // Create each neighbour along the vertex stack dimensions. for (std::size_t i = 0; i < coordinates.size()-1; ++i) { - int x = coordinates[i]; + const std::size_t x = coordinates[i]; if (m_bundle.stacks[i]->size() > x+1) { ++tmpCoordinates[i]; CreateNeighbour(tmpCoordinates); @@ -64,7 +64,7 @@ void Cube::CreateNeighbours(const std::vector<int> &coordinates) } } // Create the neighbour along the translation dimension. - int x = coordinates.back(); + const std::size_t x = coordinates.back(); if (m_bundle.translations->GetSize() > x+1) { ++tmpCoordinates.back(); CreateNeighbour(tmpCoordinates); diff --git a/moses/Syntax/F2S/HyperTreeLoader.cpp b/moses/Syntax/F2S/HyperTreeLoader.cpp index 0ffc3602f..f3caa2cec 100644 --- a/moses/Syntax/F2S/HyperTreeLoader.cpp +++ b/moses/Syntax/F2S/HyperTreeLoader.cpp @@ -3,7 +3,7 @@ #include <sys/stat.h> #include <cmath> -#include <stdlib.h> +#include <cstdlib> #include <fstream> #include <string> #include <iterator> diff --git a/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h b/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h index d7ce2cf7c..74f2347a6 100644 --- a/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h +++ b/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h @@ -190,8 +190,9 @@ int RuleMatcherHyperTree<Callback>::SubSeqLength(const HyperPath::NodeSeq &seq, int pos) { int length = 0; - while (pos != seq.size() && seq[pos] != HyperPath::kComma) { - ++pos; + HyperPath::NodeSeq::size_type curpos = pos; + while (curpos != seq.size() && seq[curpos] != HyperPath::kComma) { + ++curpos; ++length; } return length; diff --git a/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h b/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h index f50cee3a0..24135c734 100644 --- a/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h +++ b/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h @@ -163,7 +163,7 @@ void Scope3Parser<Callback>::RecordPatternApplicationSpans( for (int j = std::max(e1, i+minSpan-1); j <= e2; ++j) { std::size_t span = j-i+1; assert(span >= 1); - if (span < minSpan) { + if (span < std::size_t(minSpan)) { continue; } if (m_maxChartSpan && span > m_maxChartSpan) { diff --git a/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h b/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h index 4e48429ab..4f815c78d 100644 --- a/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h +++ b/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h @@ -58,7 +58,7 @@ private: const int absStart = m_ranges[0].minStart + offset; const int minWidth = std::max(1, range.minEnd - absStart + 1); - const int maxWidth = range.maxEnd - absStart + 1; + const std::size_t maxWidth = range.maxEnd - absStart + 1; const std::vector<std::vector<const PVertex *> > &innerVec = m_lattice[offset][nonTermIndex+1]; diff --git a/moses/Syntax/S2T/RuleTrieLoader.cpp b/moses/Syntax/S2T/RuleTrieLoader.cpp index b9f7484ad..a88c0f5fe 100644 --- a/moses/Syntax/S2T/RuleTrieLoader.cpp +++ b/moses/Syntax/S2T/RuleTrieLoader.cpp @@ -1,14 +1,14 @@ #include "RuleTrieLoader.h" #include <sys/stat.h> -#include <stdlib.h> +#include <cstdlib> #include <fstream> #include <string> #include <iterator> #include <algorithm> #include <iostream> -#include <math.h> +#include <cmath> #include "moses/FactorCollection.h" #include "moses/Word.h" diff --git a/moses/TargetPhrase.cpp b/moses/TargetPhrase.cpp index 70568940d..d6abd16ff 100644 --- a/moses/TargetPhrase.cpp +++ b/moses/TargetPhrase.cpp @@ -20,7 +20,7 @@ ***********************************************************************/ #include <algorithm> -#include <stdlib.h> +#include <cstdlib> #include "util/exception.hh" #include "util/tokenize_piece.hh" diff --git a/moses/ThreadPool.cpp b/moses/ThreadPool.cpp index 1113cef9d..cba459697 100644 --- a/moses/ThreadPool.cpp +++ b/moses/ThreadPool.cpp @@ -57,9 +57,8 @@ void ThreadPool::Execute() if (task) { // must read from task before run. otherwise task may be deleted by main thread // race condition - bool del = task->DeleteAfterExecution(); + task->DeleteAfterExecution(); task->Run(); - // if (del) delete task; // not needed any more, since we use shared ptrs. } m_threadAvailable.notify_all(); } while (!m_stopped); diff --git a/moses/TranslationModel/CompactPT/BlockHashIndex.h b/moses/TranslationModel/CompactPT/BlockHashIndex.h index f8d526930..b3f5e6f4b 100644 --- a/moses/TranslationModel/CompactPT/BlockHashIndex.h +++ b/moses/TranslationModel/CompactPT/BlockHashIndex.h @@ -38,7 +38,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #ifdef WITH_THREADS #include "moses/ThreadPool.h" #else -#include <time.h> +#include <ctime> #endif #include <boost/shared_ptr.hpp> diff --git a/moses/TranslationModel/CompactPT/MurmurHash3.cpp b/moses/TranslationModel/CompactPT/MurmurHash3.cpp index 4e4cd4258..fb6946bbc 100644 --- a/moses/TranslationModel/CompactPT/MurmurHash3.cpp +++ b/moses/TranslationModel/CompactPT/MurmurHash3.cpp @@ -18,7 +18,7 @@ #define FORCE_INLINE __forceinline
-#include <stdlib.h>
+#include <cstdlib>
#define ROTL32(x,y) _rotl(x,y)
#define ROTL64(x,y) _rotl64(x,y)
diff --git a/moses/TranslationModel/DynSAInclude/FileHandler.cpp b/moses/TranslationModel/DynSAInclude/FileHandler.cpp index 8645833fe..9413ffd7c 100644 --- a/moses/TranslationModel/DynSAInclude/FileHandler.cpp +++ b/moses/TranslationModel/DynSAInclude/FileHandler.cpp @@ -1,5 +1,5 @@ #include "FileHandler.h" -#include <stdio.h> +#include <cstdio> #ifdef WIN32 #define popen(A, B) _popen(A, B) diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h index f80c3febd..667b12bad 100644 --- a/moses/TranslationModel/PhraseDictionary.h +++ b/moses/TranslationModel/PhraseDictionary.h @@ -36,7 +36,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <boost/thread/tss.hpp> #else #include <boost/scoped_ptr.hpp> -#include <time.h> +#include <ctime> #endif #include "moses/Phrase.h" diff --git a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp index a336da759..69b7e9f5f 100644 --- a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp +++ b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp @@ -1,5 +1,7 @@ // vim:tabstop=2 -#include <stdlib.h> +#include <cstdlib> +#include <boost/filesystem.hpp> + #include "PhraseDictionaryTransliteration.h" #include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" #include "moses/DecodeGraph.h" @@ -68,10 +70,9 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &input inputPath.SetTargetPhrases(*this, tpColl, NULL); } else { // TRANSLITERATE - char *ptr = tmpnam(NULL); - string inFile(ptr); - ptr = tmpnam(NULL); - string outDir(ptr); + const boost::filesystem::path + inFile = boost::filesystem::unique_path(), + outDir = boost::filesystem::unique_path(); ofstream inStream(inFile.c_str()); inStream << sourcePhrase.ToString() << endl; @@ -83,14 +84,14 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &input " --external-bin-dir " + m_externalDir + " --input-extension " + m_inputLang + " --output-extension " + m_outputLang + - " --oov-file " + inFile + - " --out-dir " + outDir; + " --oov-file " + inFile.native() + + " --out-dir " + outDir.native(); int ret = system(cmd.c_str()); UTIL_THROW_IF2(ret != 0, "Transliteration script error"); TargetPhraseCollection *tpColl = new TargetPhraseCollection(); - vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir); + vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir.native()); vector<TargetPhrase*>::const_iterator iter; for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) { TargetPhrase *tp = *iter; @@ -104,9 +105,7 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &input // clean up temporary files remove(inFile.c_str()); - - cmd = "rm -rf " + outDir; - system(cmd.c_str()); + boost::filesystem::remove_all(outDir); } } diff --git a/moses/TranslationModel/ProbingPT/huffmanish.hh b/moses/TranslationModel/ProbingPT/huffmanish.hh index 46b7dbeea..14db53a69 100644 --- a/moses/TranslationModel/ProbingPT/huffmanish.hh +++ b/moses/TranslationModel/ProbingPT/huffmanish.hh @@ -3,7 +3,7 @@ //Huffman encodes a line and also produces the vocabulary ids #include "hash.hh" #include "line_splitter.hh" -#include <stdio.h> +#include <cstdio> #include <fstream> #include <iostream> #include <sstream> diff --git a/moses/TranslationModel/ProbingPT/line_splitter.hh b/moses/TranslationModel/ProbingPT/line_splitter.hh index c699a28c0..36833616f 100644 --- a/moses/TranslationModel/ProbingPT/line_splitter.hh +++ b/moses/TranslationModel/ProbingPT/line_splitter.hh @@ -4,7 +4,7 @@ #include "util/tokenize_piece.hh" #include "util/file_piece.hh" #include <vector> -#include <stdlib.h> //atof +#include <cstdlib> //atof #include "util/string_piece.hh" //Tokenization and work with StringPiece #include "util/tokenize_piece.hh" #include <vector> diff --git a/moses/TranslationModel/ProbingPT/storing.hh b/moses/TranslationModel/ProbingPT/storing.hh index 821fd14ca..eb3b1ea53 100644 --- a/moses/TranslationModel/ProbingPT/storing.hh +++ b/moses/TranslationModel/ProbingPT/storing.hh @@ -1,6 +1,6 @@ #pragma once -#include <stdio.h> +#include <cstdio> #include <fstream> #include <iostream> diff --git a/moses/TranslationModel/ProbingPT/tests/tokenization_tests.cpp b/moses/TranslationModel/ProbingPT/tests/tokenization_tests.cpp deleted file mode 100644 index 528c9c37c..000000000 --- a/moses/TranslationModel/ProbingPT/tests/tokenization_tests.cpp +++ /dev/null @@ -1,206 +0,0 @@ -#include "line_splitter.hh" - -bool test_vectorinsert() -{ - StringPiece line1 = StringPiece("! ! ! ! ||| ! ! ! ! ||| 0.0804289 0.141656 0.0804289 0.443409 2.718 ||| 0-0 1-1 2-2 3-3 ||| 1 1 1"); - StringPiece line2 = StringPiece("! ! ! ) , has ||| ! ! ! ) - , a ||| 0.0804289 0.0257627 0.0804289 0.00146736 2.718 ||| 0-0 1-1 2-2 3-3 4-4 4-5 5-6 ||| 1 1 1"); - line_text output = splitLine(line1); - line_text output2 = splitLine(line2); - - //Init container vector and iterator. - std::vector<char> container; - container.reserve(10000); //Reserve vector - std::vector<char>::iterator it = container.begin(); - std::pair<std::vector<char>::iterator, int> binary_append_ret; //Return values from vector_append - - //Put a value into the vector - binary_append_ret = vector_append(&output, &container, it, false); - it = binary_append_ret.first; - binary_append_ret = vector_append(&output2, &container, it, false); - it = binary_append_ret.first; - - std::string test(container.begin(), container.end()); - std::string should_be = "! ! ! ! 0.0804289 0.141656 0.0804289 0.443409 2.718 0-0 1-1 2-2 3-3 1 1 1! ! ! ) - , a 0.0804289 0.0257627 0.0804289 0.00146736 2.718 0-0 1-1 2-2 3-3 4-4 4-5 5-6 1 1 1"; - if (test == should_be) { - return true; - } else { - return false; - } -} - -bool probabilitiesTest() -{ - StringPiece line1 = StringPiece("0.536553 0.75961 0.634108 0.532927 2.718"); - StringPiece line2 = StringPiece("1.42081e-05 3.91895e-09 0.0738539 0.749514 2.718"); - - std::vector<double> pesho; - bool peshobool = false; - bool kirobool = false; - std::vector<double> kiro; - - pesho = splitProbabilities(line1); - kiro = splitProbabilities(line2); - - if (pesho[0] == 0.536553 && pesho[1] == 0.75961 && pesho[2] == 0.634108 && pesho[3] == 0.532927 && pesho[4] == 2.718 && pesho.size() == 5) { - peshobool = true; - } else { - std::cout << "Processed: " << pesho[0] << " " << pesho[1] << " " << pesho[2] << " " << pesho[3] << " " << pesho[4] << std::endl; - std::cout << "Size is: " << pesho.size() << " Expected 5." << std::endl; - std::cout << "Expected: " << "0.536553 0.75961 0.634108 0.532927 2.718" << std::endl; - } - - if (kiro[0] == 1.42081e-05 && kiro[1] == 3.91895e-09 && kiro[2] == 0.0738539 && kiro[3] == 0.749514 && kiro[4] == 2.718 && kiro.size() == 5) { - kirobool = true; - } else { - std::cout << "Processed: " << kiro[0] << " " << kiro[1] << " " << kiro[2] << " " << kiro[3] << " " << kiro[4] << std::endl; - std::cout << "Size is: " << kiro.size() << " Expected 5." << std::endl; - std::cout << "Expected: " << "1.42081e-05 3.91895e-09 0.0738539 0.749514 2.718" << std::endl; - } - - return (peshobool && kirobool); -} - -bool wordAll1test() -{ - StringPiece line1 = StringPiece("2-0 3-1 4-2 5-2"); - StringPiece line2 = StringPiece("0-0 1-1 2-2 3-3 4-3 6-4 5-5"); - - std::vector<int> pesho; - bool peshobool = false; - bool kirobool = false; - std::vector<int> kiro; - - pesho = splitWordAll1(line1); - kiro = splitWordAll1(line2); - - if (pesho[0] == 2 && pesho[1] == 0 && pesho[2] == 3 && pesho[3] == 1 && pesho[4] == 4 - && pesho[5] == 2 && pesho[6] == 5 && pesho[7] == 2 && pesho.size() == 8) { - peshobool = true; - } else { - std::cout << "Processed: " << pesho[0] << "-" << pesho[1] << " " << pesho[2] << "-" << pesho[3] << " " - << pesho[4] << "-" << pesho[5] << " " << pesho[6] << "-" << pesho[7] << std::endl; - std::cout << "Size is: " << pesho.size() << " Expected: 8." << std::endl; - std::cout << "Expected: " << "2-0 3-1 4-2 5-2" << std::endl; - } - - if (kiro[0] == 0 && kiro[1] == 0 && kiro[2] == 1 && kiro[3] == 1 && kiro[4] == 2 && kiro[5] == 2 - && kiro[6] == 3 && kiro[7] == 3 && kiro[8] == 4 && kiro[9] == 3 && kiro[10] == 6 && kiro[11] == 4 - && kiro[12] == 5 && kiro[13] == 5 && kiro.size() == 14) { - kirobool = true; - } else { - std::cout << "Processed: " << kiro[0] << "-" << kiro[1] << " " << kiro[2] << "-" << kiro[3] << " " - << kiro[4] << "-" << kiro[5] << " " << kiro[6] << "-" << kiro[7] << " " << kiro[8] << "-" << kiro[9] - << " " << kiro[10] << "-" << kiro[11] << " " << kiro[12] << "-" << kiro[13] << std::endl; - std::cout << "Size is: " << kiro.size() << " Expected: 14" << std::endl; - std::cout << "Expected: " << "0-0 1-1 2-2 3-3 4-3 6-4 5-5" << std::endl; - } - - return (peshobool && kirobool); -} - -bool wordAll2test() -{ - StringPiece line1 = StringPiece("4 9 1"); - StringPiece line2 = StringPiece("3255 9 1"); - - std::vector<int> pesho; - bool peshobool = false; - bool kirobool = false; - std::vector<int> kiro; - - pesho = splitWordAll2(line1); - kiro = splitWordAll2(line2); - - if (pesho[0] == 4 && pesho[1] == 9 && pesho[2] == 1 && pesho.size() == 3) { - peshobool = true; - } else { - std::cout << "Processed: " << pesho[0] << " " << pesho[1] << " " << pesho[2] << std::endl; - std::cout << "Size: " << pesho.size() << " Expected: 3" << std::endl; - std::cout << "Expected: " << "4 9 1" << std::endl; - } - - if (kiro[0] == 3255 && kiro[1] == 9 && kiro[2] == 1 && kiro.size() == 3) { - kirobool = true; - } else { - std::cout << "Processed: " << kiro[0] << " " << kiro[1] << " " << kiro[2] << std::endl; - std::cout << "Size: " << kiro.size() << " Expected: 3" << std::endl; - std::cout << "Expected: " << "3255 9 1" << std::endl; - } - - return (peshobool && kirobool); - -} - -bool test_tokenization() -{ - StringPiece line1 = StringPiece("! ! ! ! ||| ! ! ! ! ||| 0.0804289 0.141656 0.0804289 0.443409 2.718 ||| 0-0 1-1 2-2 3-3 ||| 1 1 1"); - StringPiece line2 = StringPiece("! ! ! ) , has ||| ! ! ! ) - , a ||| 0.0804289 0.0257627 0.0804289 0.00146736 2.718 ||| 0-0 1-1 2-2 3-3 4-4 4-5 5-6 ||| 1 1 1"); - StringPiece line3 = StringPiece("! ! ! ) , ||| ! ! ! ) - , ||| 0.0804289 0.075225 0.0804289 0.00310345 2.718 ||| 0-0 1-1 2-2 3-3 4-4 4-5 ||| 1 1 1"); - StringPiece line4 = StringPiece("! ! ! ) ||| ! ! ! ) . ||| 0.0804289 0.177547 0.0268096 0.000872597 2.718 ||| 0-0 1-1 2-2 3-3 ||| 1 3 1"); - - line_text output1 = splitLine(line1); - line_text output2 = splitLine(line2); - line_text output3 = splitLine(line3); - line_text output4 = splitLine(line4); - - bool test1 = output1.prob == StringPiece("0.0804289 0.141656 0.0804289 0.443409 2.718"); - bool test2 = output2.word_all1 == StringPiece("0-0 1-1 2-2 3-3 4-4 4-5 5-6"); - bool test3 = output2.target_phrase == StringPiece("! ! ! ) - , a"); - bool test4 = output3.source_phrase == StringPiece("! ! ! ) ,"); - bool test5 = output4.word_all2 == StringPiece("1 3 1"); - - //std::cout << test1 << " " << test2 << " " << test3 << " " << test4 << std::endl; - - return (test1 && test2 && test3 && test4 && test5); - -} - -bool test_linesplitter() -{ - StringPiece line1 = StringPiece("! ] 0.0738539 0.901133 0.0738539 0.65207 2.718 0-0 1-1 1 1 1"); - target_text ans1; - ans1 = splitSingleTargetLine(line1); - - /* For testing purposes - std::cout << ans1.target_phrase[0] << " " <<ans1.target_phrase[1] << " Size: " << ans1.target_phrase.size() << std::endl; - std::cout << ans1.word_all1[3] << " " << ans1.word_all2[2] << " " << ans1.prob[3] << std::endl; */ - - return (ans1.target_phrase.size() == 2 && ans1.prob.size() == 5 && ans1.word_all1.size() == 4 && ans1.word_all2.size() == 3); -} - -bool test_linessplitter() -{ - StringPiece line1 = StringPiece("! ] 0.0738539 0.901133 0.0738539 0.65207 2.718 0-0 1-1 1 1 1\n\n! ) . proto došlo 0.0738539 7.14446e-06"); - StringPiece line2 = StringPiece("! " ) 0.536553 0.75961 0.634108 0.532927 2.718 0-0 1-1 2-2 13 11 8\n! ) . 0.0369269 0.00049839 0.00671399 0.00372884 2.718 0-0 1-1 2-1 2-2 2 11 1\n" ! ) 0.0738539 0.75961 0.00671399 0.532927 2.718 1-0 0-1 2-2 1 11 1\nse ! " ) 0.0738539 0.75961 0.00671399 0.0225211 2.718 0-1 1-2 2-3 1 11 1\n\n! " , a to 0.0738539 0.0894238 0.0738539 0.048"); - - std::vector<target_text> ans1; - std::vector<target_text> ans2; - - ans1 = splitTargetLine(line1); - ans2 = splitTargetLine(line2); - - bool sizes = ans1.size() == 1 && ans2.size() == 4; - bool prob = ans1[0].prob[3] == 0.65207 && ans2[1].prob[1] == 0.00049839; - bool word_alls = ans2[0].word_all2[1] == 11 && ans2[3].word_all1[5] == 3; - - /* FOr testing - std::cout << ans1.size() << std::endl; - std::cout << ans2.size() << std::endl; - std::cout << ans1[0].prob[3] << std::endl; - std::cout << ans2[1].prob[1] << std::endl; - std::cout << ans2[0].word_all2[1] << std::endl; - std::cout << ans2[3].word_all1[5] << std::endl; */ - - return sizes && prob && word_alls; -} - -int main() -{ - if (probabilitiesTest() && wordAll1test() && wordAll2test() && test_tokenization() && test_linesplitter() && test_linessplitter() && test_vectorinsert()) { - std::cout << "All tests pass!" << std::endl; - } else { - std::cout << "Failiure in some tests!" << std::endl; - } - - return 1; -}
\ No newline at end of file diff --git a/moses/TranslationModel/ProbingPT/tests/vocabid_test.cpp b/moses/TranslationModel/ProbingPT/tests/vocabid_test.cpp deleted file mode 100644 index fac439eeb..000000000 --- a/moses/TranslationModel/ProbingPT/tests/vocabid_test.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include <map> //Map for vocab ids - -#include "hash.hh" -#include "vocabid.hh" - -int main(int argc, char* argv[]) -{ - - //Create a map and serialize it - std::map<uint64_t, std::string> vocabids; - StringPiece demotext = StringPiece("Demo text with 3 elements"); - add_to_map(&vocabids, demotext); - //Serialize map - serialize_map(&vocabids, "/tmp/testmap.bin"); - - //Read the map and test if the values are the same - std::map<uint64_t, std::string> newmap; - read_map(&newmap, "/tmp/testmap.bin"); - - //Used hashes - uint64_t num1 = getHash(StringPiece("Demo")); - uint64_t num2 = getVocabID("text"); - uint64_t num3 = getHash(StringPiece("with")); - uint64_t num4 = getVocabID("3"); - uint64_t num5 = getHash(StringPiece("elements")); - uint64_t num6 = 0; - - //Tests - bool test1 = getStringFromID(&newmap, num1) == getStringFromID(&vocabids, num1); - bool test2 = getStringFromID(&newmap, num2) == getStringFromID(&vocabids, num2); - bool test3 = getStringFromID(&newmap, num3) == getStringFromID(&vocabids, num3); - bool test4 = getStringFromID(&newmap, num4) == getStringFromID(&vocabids, num4); - bool test5 = getStringFromID(&newmap, num5) == getStringFromID(&vocabids, num5); - bool test6 = getStringFromID(&newmap, num6) == getStringFromID(&vocabids, num6); - - - if (test1 && test2 && test3 && test4 && test5 && test6) { - std::cout << "Map was successfully written and read!" << std::endl; - } else { - std::cout << "Error! " << test1 << " " << test2 << " " << test3 << " " << test4 << " " << test5 << " " << test6 << std::endl; - } - - - return 1; - -} diff --git a/moses/TranslationModel/RuleTable/LoaderStandard.cpp b/moses/TranslationModel/RuleTable/LoaderStandard.cpp index 95463feea..f9e6ac6fd 100644 --- a/moses/TranslationModel/RuleTable/LoaderStandard.cpp +++ b/moses/TranslationModel/RuleTable/LoaderStandard.cpp @@ -25,7 +25,7 @@ #include <algorithm> #include <iostream> #include <sys/stat.h> -#include <stdlib.h> +#include <cstdlib> #include <boost/algorithm/string/predicate.hpp> #include "Trie.h" #include "moses/FactorCollection.h" diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp index 0928cf1e4..1ca9dce67 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp @@ -19,10 +19,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <limits.h> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <climits> #include <sys/types.h> #include <unistd.h> #include <dirent.h> @@ -50,7 +50,7 @@ using namespace std; #if defined __MINGW32__ && !defined mkdtemp #include <windows.h> -#include <errno.h> +#include <cerrno> char *mkdtemp(char *tempbuf) { int rand_value = 0; @@ -392,10 +392,10 @@ TO_STRING_BODY(PhraseDictionaryFuzzyMatch); // friend ostream& operator<<(ostream& out, const PhraseDictionaryFuzzyMatch& phraseDict) { + /* typedef PhraseDictionaryNodeMemory::TerminalMap TermMap; typedef PhraseDictionaryNodeMemory::NonTerminalMap NonTermMap; - /* const PhraseDictionaryNodeMemory &coll = phraseDict.m_collection; for (NonTermMap::const_iterator p = coll.m_nonTermMap.begin(); p != coll.m_nonTermMap.end(); ++p) { const Word &sourceNonTerm = p->first.first; diff --git a/moses/TranslationModel/UG/mm/tpt_tightindex.cc b/moses/TranslationModel/UG/mm/tpt_tightindex.cc index ff7153af7..da28c6d93 100644 --- a/moses/TranslationModel/UG/mm/tpt_tightindex.cc +++ b/moses/TranslationModel/UG/mm/tpt_tightindex.cc @@ -17,7 +17,7 @@ // #include <iostream> -#include <assert.h> +#include <cassert> #include "tpt_tightindex.h" namespace ugdiss diff --git a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc index 2273fefa7..c6704beac 100644 --- a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc +++ b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc @@ -1,7 +1,7 @@ // -*- c++ -*- // (c) 2007-2013 Ulrich Germann #include <sstream> -#include <string.h> +#include <cstring> #include <algorithm> #include <iostream> #include <stdexcept> diff --git a/moses/TranslationModel/fuzzy-match/Vocabulary.h b/moses/TranslationModel/fuzzy-match/Vocabulary.h index 5a79e2f26..f5245ebe3 100644 --- a/moses/TranslationModel/fuzzy-match/Vocabulary.h +++ b/moses/TranslationModel/fuzzy-match/Vocabulary.h @@ -4,8 +4,8 @@ #include <iostream> #include <fstream> -#include <assert.h> -#include <stdlib.h> +#include <cassert> +#include <cstdlib> #include <string> #include <queue> #include <map> diff --git a/moses/Util.cpp b/moses/Util.cpp index 1d1df7d58..a8175b58d 100644 --- a/moses/Util.cpp +++ b/moses/Util.cpp @@ -29,7 +29,7 @@ #include <cstring> #include <cctype> #include <algorithm> -#include <stdio.h> +#include <cstdio> #include <iostream> #include <iomanip> #include <boost/algorithm/string/predicate.hpp> diff --git a/moses/mbr.cpp b/moses/mbr.cpp index 6a8dfa823..df2313b66 100644 --- a/moses/mbr.cpp +++ b/moses/mbr.cpp @@ -4,10 +4,10 @@ #include <iomanip> #include <vector> #include <map> -#include <stdlib.h> -#include <math.h> +#include <cstdlib> +#include <cmath> #include <algorithm> -#include <stdio.h> +#include <cstdio> #include "moses/TrellisPathList.h" #include "moses/TrellisPath.h" #include "moses/StaticData.h" |