diff options
author | Marcin Junczys-Dowmunt <junczys@amu.edu.pl> | 2015-01-08 14:30:07 +0300 |
---|---|---|
committer | Marcin Junczys-Dowmunt <junczys@amu.edu.pl> | 2015-01-08 14:30:07 +0300 |
commit | 3f1cb319cd696660c1c24ba682a0399b41dfc7c4 (patch) | |
tree | 9cba9b8a0d3b002a5cd332be780a2e7344ef7b9f /moses/TabbedSentence.cpp | |
parent | 6b73d1b8f4d6638fbf2d0e41b83cc794c94a2a0e (diff) |
Added TabbedSentence and first attempt at external VW features
Diffstat (limited to 'moses/TabbedSentence.cpp')
-rw-r--r-- | moses/TabbedSentence.cpp | 72 |
1 files changed, 72 insertions, 0 deletions
diff --git a/moses/TabbedSentence.cpp b/moses/TabbedSentence.cpp new file mode 100644 index 000000000..471a8cc7d --- /dev/null +++ b/moses/TabbedSentence.cpp @@ -0,0 +1,72 @@ +// $Id$ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <vector> +#include <string> +#include <sstream> +#include <boost/algorithm/string.hpp> + +#include "TabbedSentence.h" + +namespace Moses +{ + + void TabbedSentence::CreateFromString(const std::vector<FactorType> &factorOrder + , const std::string &tabbedString) { + TabbedColumns allColumns; + + boost::split(allColumns, tabbedString, boost::is_any_of("\t")); + + if(allColumns.size() < 2) { + Sentence::CreateFromString(factorOrder, tabbedString); + } + else { + m_columns.resize(allColumns.size() - 1); + std::copy(allColumns.begin() + 1, allColumns.end(), m_columns.begin()); + Sentence::CreateFromString(factorOrder, allColumns[0]); + } + } + + int TabbedSentence::Read(std::istream& in, const std::vector<FactorType>& factorOrder) { + TabbedColumns allColumns; + + std::string line; + if (getline(in, line, '\n').eof()) + return 0; + + boost::split(allColumns, line, boost::is_any_of("\t")); + + if(allColumns.size() < 2) { + std::stringstream dummyStream; + dummyStream << line << std::endl; + return Sentence::Read(dummyStream, factorOrder); + } + else { + m_columns.resize(allColumns.size() - 1); + std::copy(allColumns.begin() + 1, allColumns.end(), m_columns.begin()); + + std::stringstream dummyStream; + dummyStream << allColumns[0] << std::endl; + return Sentence::Read(dummyStream, factorOrder); + } + } + +} |