diff options
Diffstat (limited to 'moses/TranslationModel/ProbingPT/line_splitter.hh')
-rw-r--r-- | moses/TranslationModel/ProbingPT/line_splitter.hh | 36 |
1 files changed, 31 insertions, 5 deletions
diff --git a/moses/TranslationModel/ProbingPT/line_splitter.hh b/moses/TranslationModel/ProbingPT/line_splitter.hh index 2cb9a3c8c..cec0a5f45 100644 --- a/moses/TranslationModel/ProbingPT/line_splitter.hh +++ b/moses/TranslationModel/ProbingPT/line_splitter.hh @@ -9,8 +9,12 @@ #include "util/tokenize_piece.hh" #include <vector> +namespace Moses +{ + //Struct for holding processed line -struct line_text { +struct line_text +{ StringPiece source_phrase; StringPiece target_phrase; StringPiece prob; @@ -18,16 +22,38 @@ struct line_text { StringPiece counts; StringPiece sparse_score; StringPiece property; + std::string property_to_be_binarized; }; //Struct for holding processed line -struct target_text { +struct target_text +{ std::vector<unsigned int> target_phrase; std::vector<float> prob; - std::vector<unsigned char> word_all1; + std::vector<size_t> word_align_term; + std::vector<size_t> word_align_non_term; + std::vector<char> counts; + std::vector<char> sparse_score; + std::vector<char> property; + + /* + void Reset() + { + target_phrase.clear(); + prob.clear(); + word_all1.clear(); + counts.clear(); + sparse_score.clear(); + property.clear(); + } + */ }; //Ask if it's better to have it receive a pointer to a line_text struct -line_text splitLine(StringPiece textin); +line_text splitLine(const StringPiece &textin, bool scfg); +void reformatSCFG(line_text &output); + +std::vector<unsigned char> splitWordAll1(const StringPiece &textin); + +} -std::vector<unsigned char> splitWordAll1(StringPiece textin); |