diff options
author | akimbal1 <akimball2@bloomberg.net> | 2015-04-02 21:15:15 +0300 |
---|---|---|
committer | akimbal1 <akimball2@bloomberg.net> | 2015-04-02 21:15:15 +0300 |
commit | ad70c9a35d08cf27a4c8c68189953328953759dd (patch) | |
tree | 7c3a235a8d1bdf6ae58075978ad0411bb0451c90 /contrib | |
parent | b4e24a2fb85e7e0a3e514665e4a90d6963c6b3c2 (diff) | |
parent | d71e516176ee3c828439abb9ec8455300915f651 (diff) |
resolve conflicts
Diffstat (limited to 'contrib')
-rw-r--r-- | contrib/c++tokenizer/tokenizer.cpp | 20 | ||||
-rw-r--r-- | contrib/other-builds/manual-label/EnOpenNLPChunker.cpp | 21 |
2 files changed, 20 insertions, 21 deletions
diff --git a/contrib/c++tokenizer/tokenizer.cpp b/contrib/c++tokenizer/tokenizer.cpp index f8caf91ad..5fccdb02f 100644 --- a/contrib/c++tokenizer/tokenizer.cpp +++ b/contrib/c++tokenizer/tokenizer.cpp @@ -1589,20 +1589,20 @@ Tokenizer::tokenize(std::istream& is, std::ostream& os) results[ithread].resize(line_pos); break; } - lines[ithread][line_pos].clear(); + //lines[ithread][line_pos].clear(); TODO clang error } else if (skip_xml_p && (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) { - lines[ithread][line_pos].clear(); + //lines[ithread][line_pos].clear(); TODO clang error } else { - lines[ithread][line_pos] = - std::string(SPC_BYTE).append(istr).append(SPC_BYTE); + //lines[ithread][line_pos] = + // std::string(SPC_BYTE).append(istr).append(SPC_BYTE); TODO clang error } } - if (line_pos) - workers[ithread] = - boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread])); - + if (line_pos) { + //workers[ithread] = + // boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread])); TODO clang error + } } // end for loop starting threads for (std::size_t ithread = 0; ithread < nthreads; ++ithread) { @@ -1776,8 +1776,8 @@ Tokenizer::splitter(const std::string &istr, bool *continuation_ptr) { charclass_t prev_class = empty; charclass_t curr_class = empty; - charclass_t seq[SEQ_LIM] = { empty }; - std::size_t pos[SEQ_LIM] = { 0 }; + std::vector<charclass_t> seq(SEQ_LIM, empty); + std::vector<std::size_t> pos(SEQ_LIM, 0); std::size_t seqpos = 0; GUnicodeType curr_type = G_UNICODE_UNASSIGNED; diff --git a/contrib/other-builds/manual-label/EnOpenNLPChunker.cpp b/contrib/other-builds/manual-label/EnOpenNLPChunker.cpp index e3b913825..538aa9746 100644 --- a/contrib/other-builds/manual-label/EnOpenNLPChunker.cpp +++ b/contrib/other-builds/manual-label/EnOpenNLPChunker.cpp @@ -9,6 +9,7 @@ #include <algorithm> #include <fstream> #include <boost/algorithm/string/predicate.hpp> +#include <boost/filesystem.hpp> #include "EnOpenNLPChunker.h" #include "moses/Util.h" @@ -28,10 +29,11 @@ EnOpenNLPChunker::~EnOpenNLPChunker() { void EnOpenNLPChunker::Process(std::istream &in, std::ostream &out, const vector<string> &filterList) { + const boost::filesystem::path + inPath = boost::filesystem::unique_path(), + outPath = boost::filesystem::unique_path(); // read all input to a temp file - char *ptr = tmpnam(NULL); - string inStr(ptr); - ofstream inFile(ptr); + ofstream inFile(inPath.c_str()); string line; while (getline(in, line)) { @@ -40,21 +42,18 @@ void EnOpenNLPChunker::Process(std::istream &in, std::ostream &out, const vector } inFile.close(); - ptr = tmpnam(NULL); - string outStr(ptr); - // execute chunker - string cmd = "cat " + inStr + " | " + string cmd = "cat " + inPath.native() + " | " + m_openNLPPath + "/bin/opennlp POSTagger " + m_openNLPPath + "/models/en-pos-maxent.bin | " + m_openNLPPath + "/bin/opennlp ChunkerME " + m_openNLPPath + "/models/en-chunker.bin > " - + outStr; + + outPath.native(); //g << "Executing:" << cmd << endl; int ret = system(cmd.c_str()); // read result of chunker and output as Moses xml trees - ifstream outFile(outStr.c_str()); + ifstream outFile(outPath.c_str()); size_t lineNum = 0; while (getline(outFile, line)) { @@ -66,8 +65,8 @@ void EnOpenNLPChunker::Process(std::istream &in, std::ostream &out, const vector outFile.close(); // clean up temporary files - remove(inStr.c_str()); - remove(outStr.c_str()); + remove(inPath.c_str()); + remove(outPath.c_str()); } void EnOpenNLPChunker::MosesReformat(const string &line, std::ostream &out, const vector<string> &filterList) |