diff options
332 files changed, 6017 insertions, 2498 deletions
diff --git a/.gitignore b/.gitignore index 388da9823..77537f590 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ mingw/MosesGUI/_eric4project/ contrib/m4m/merge-sorted mert/hgdecode +.bash_history* diff --git a/.gitmodules b/.gitmodules index 51ab8750b..9339fa35a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "contrib/omtc/omtc"] path = contrib/omtc/omtc url = https://github.com/ianj-als/omtc.git +[submodule "mmt"] + path = mmt + url = https://github.com/modernmt/moses-submodule diff --git a/BUILD-INSTRUCTIONS.txt b/BUILD-INSTRUCTIONS.txt index 7b9bc3a8a..a41582bfa 100644 --- a/BUILD-INSTRUCTIONS.txt +++ b/BUILD-INSTRUCTIONS.txt @@ -7,3 +7,4 @@ into the source tree from elsewhere: * "bjam-files" is taken from Boost. * "util" and "lm" are taken from KenLM: https://github.com/kpu/kenlm + diff --git a/COPYING b/COPYING new file mode 100644 index 000000000..805dbfe12 --- /dev/null +++ b/COPYING @@ -0,0 +1,460 @@ + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations +below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it +becomes a de-facto standard. To achieve this, non-free programs must +be allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control +compilation and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at least + three years, to give the same user the materials specified in + Subsection 6a, above, for a charge no more than the cost of + performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply, and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License +may add an explicit geographical distribution limitation excluding those +countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + @@ -17,6 +17,9 @@ #Note that, like language models, this is the --prefix where the library was #installed, not some executable within the library. # +#--no-xmlrpc-c +# Don't use xmlrpc-c library, even if it exists. Don't build moses server +# #Compact phrase table and compact lexical reordering table #--with-cmph=/path/to/cmph # @@ -51,7 +54,7 @@ # --static forces static linking (the default will fall # back to shared) # -# debug-symbols=on|off include (default) or exclude debugging +# debug-symbols=on|off include or exclude (default) debugging # information also known as -g # --notrace compiles without TRACE macros # @@ -143,6 +146,13 @@ if [ option.get "debug-build" : : "yes" ] { echo "Building with -Og to enable easier profiling and debugging. Only available on gcc 4.8+." ; } +if [ option.get "with-address-sanitizer" : : "yes" ] { + requirements += <cxxflags>-fsanitize=address ; + requirements += <cxxflags>-fno-omit-frame-pointer ; + requirements += <linkflags>-fsanitize=address ; + echo "Building with AddressSanitizer to enable debugging of memory errors. Only available on gcc 4.8+." ; +} + if [ option.get "enable-mpi" : : "yes" ] { import mpi ; using mpi ; @@ -288,6 +298,8 @@ contrib/server//mosesserver mm rephraser contrib/c++tokenizer//tokenizer +contrib/expected-bleu-training//train-expected-bleu +contrib/expected-bleu-training//prepare-expected-bleu-training ; diff --git a/OnDiskPt/OnDiskWrapper.cpp b/OnDiskPt/OnDiskWrapper.cpp index 3bfb7d2b1..57fae5162 100644 --- a/OnDiskPt/OnDiskWrapper.cpp +++ b/OnDiskPt/OnDiskWrapper.cpp @@ -25,6 +25,7 @@ #include "OnDiskWrapper.h" #include "moses/Factor.h" #include "util/exception.hh" +#include "util/string_stream.hh" using namespace std; @@ -223,7 +224,8 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact { bool isNonTerminal = origWord.IsNonTerminal(); Word *newWord = new Word(isNonTerminal); - stringstream strme; + + util::StringStream strme; size_t factorType = factorsVec[0]; const Moses::Factor *factor = origWord.GetFactor(factorType); diff --git a/OnDiskPt/PhraseNode.cpp b/OnDiskPt/PhraseNode.cpp index 8e50147b2..b77e8d807 100644 --- a/OnDiskPt/PhraseNode.cpp +++ b/OnDiskPt/PhraseNode.cpp @@ -249,16 +249,12 @@ size_t PhraseNode::ReadChild(Word &wordFound, uint64_t &childFilePos, const char return memRead; } -const TargetPhraseCollection *PhraseNode::GetTargetPhraseCollection(size_t tableLimit, OnDiskWrapper &onDiskWrapper) const +TargetPhraseCollection::shared_ptr +PhraseNode:: +GetTargetPhraseCollection(size_t tableLimit, OnDiskWrapper &onDiskWrapper) const { - TargetPhraseCollection *ret = new TargetPhraseCollection(); - - if (m_value > 0) - ret->ReadFromFile(tableLimit, m_value, onDiskWrapper); - else { - - } - + TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection); + if (m_value > 0) ret->ReadFromFile(tableLimit, m_value, onDiskWrapper); return ret; } diff --git a/OnDiskPt/PhraseNode.h b/OnDiskPt/PhraseNode.h index 901852952..1e3611d74 100644 --- a/OnDiskPt/PhraseNode.h +++ b/OnDiskPt/PhraseNode.h @@ -92,7 +92,10 @@ public: } const PhraseNode *GetChild(const Word &wordSought, OnDiskWrapper &onDiskWrapper) const; - const TargetPhraseCollection *GetTargetPhraseCollection(size_t tableLimit, OnDiskWrapper &onDiskWrapper) const; + + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection(size_t tableLimit, + OnDiskWrapper &onDiskWrapper) const; void AddCounts(const std::vector<float> &counts) { m_counts = counts; diff --git a/OnDiskPt/TargetPhraseCollection.cpp b/OnDiskPt/TargetPhraseCollection.cpp index 73ad2540c..8d06af163 100644 --- a/OnDiskPt/TargetPhraseCollection.cpp +++ b/OnDiskPt/TargetPhraseCollection.cpp @@ -114,23 +114,22 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper) } -Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors +Moses::TargetPhraseCollection::shared_ptr TargetPhraseCollection::ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors , const std::vector<Moses::FactorType> &outputFactors , const Moses::PhraseDictionary &phraseDict , const std::vector<float> &weightT , Vocab &vocab , bool isSyntax) const { - Moses::TargetPhraseCollection *ret = new Moses::TargetPhraseCollection(); + Moses::TargetPhraseCollection::shared_ptr ret; + ret.reset(new Moses::TargetPhraseCollection); CollType::const_iterator iter; for (iter = m_coll.begin(); iter != m_coll.end(); ++iter) { const TargetPhrase &tp = **iter; - Moses::TargetPhrase *mosesPhrase = tp.ConvertToMoses(inputFactors, outputFactors - , vocab - , phraseDict - , weightT - , isSyntax); + Moses::TargetPhrase *mosesPhrase + = tp.ConvertToMoses(inputFactors, outputFactors, vocab, + phraseDict, weightT, isSyntax); /* // debugging output diff --git a/OnDiskPt/TargetPhraseCollection.h b/OnDiskPt/TargetPhraseCollection.h index 227a0ffc2..1dd37fd4e 100644 --- a/OnDiskPt/TargetPhraseCollection.h +++ b/OnDiskPt/TargetPhraseCollection.h @@ -21,6 +21,8 @@ #include "TargetPhrase.h" #include "Vocab.h" +#include "moses/TargetPhraseCollection.h" +#include <boost/shared_ptr.hpp> namespace Moses { @@ -50,6 +52,9 @@ protected: std::string m_debugStr; public: + typedef boost::shared_ptr<TargetPhraseCollection const> shared_const_ptr; + typedef boost::shared_ptr<TargetPhraseCollection> shared_ptr; + static size_t s_sortScoreInd; TargetPhraseCollection(); @@ -69,7 +74,7 @@ public: uint64_t GetFilePos() const; - Moses::TargetPhraseCollection *ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors + Moses::TargetPhraseCollection::shared_ptr ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors , const std::vector<Moses::FactorType> &outputFactors , const Moses::PhraseDictionary &phraseDict , const std::vector<float> &weightT diff --git a/OnDiskPt/queryOnDiskPt.cpp b/OnDiskPt/queryOnDiskPt.cpp index 77576d956..1eeb65d9a 100644 --- a/OnDiskPt/queryOnDiskPt.cpp +++ b/OnDiskPt/queryOnDiskPt.cpp @@ -56,7 +56,7 @@ int main(int argc, char **argv) if (node) { // source phrase points to a bunch of rules - const TargetPhraseCollection *coll = node->GetTargetPhraseCollection(tableLimit, onDiskWrapper); + TargetPhraseCollection::shared_ptr coll = node->GetTargetPhraseCollection(tableLimit, onDiskWrapper); string str = coll->GetDebugStr(); cout << "Found " << coll->GetSize() << endl; diff --git a/contrib/expected-bleu-training/ExpectedBleuOptimizer.cpp b/contrib/expected-bleu-training/ExpectedBleuOptimizer.cpp new file mode 100644 index 000000000..1acea48f6 --- /dev/null +++ b/contrib/expected-bleu-training/ExpectedBleuOptimizer.cpp @@ -0,0 +1,223 @@ +/* + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + + +#include "ExpectedBleuOptimizer.h" + + +namespace ExpectedBleuTraining +{ + + +void ExpectedBleuOptimizer::AddTrainingInstance(const size_t nBestSizeCount, + const std::vector<float>& sBleu, + const std::vector<double>& overallScoreUntransformed, + const std::vector< boost::unordered_map<size_t, float> > &sparseScore, + bool maintainUpdateSet) +{ + + // compute xBLEU + double sumUntransformedScores = 0.0; + for (std::vector<double>::const_iterator overallScoreUntransformedIt=overallScoreUntransformed.begin(); + overallScoreUntransformedIt!=overallScoreUntransformed.end(); ++overallScoreUntransformedIt) + { + sumUntransformedScores += *overallScoreUntransformedIt; + } + + double xBleu = 0.0; + assert(nBestSizeCount == overallScoreUntransformed.size()); + std::vector<double> p; + for (size_t i=0; i<nBestSizeCount; ++i) + { + if (sumUntransformedScores != 0) { + p.push_back( overallScoreUntransformed[i] / sumUntransformedScores ); + } else { + p.push_back( 0 ); + } + xBleu += p.back() * sBleu[ i ]; + } + + for (size_t i=0; i<nBestSizeCount; ++i) + { + double D = sBleu[ i ] - xBleu; + for (boost::unordered_map<size_t, float>::const_iterator sparseScoreIt=sparseScore[i].begin(); + sparseScoreIt!=sparseScore[i].end(); ++sparseScoreIt) + { + const size_t name = sparseScoreIt->first; + float N = sparseScoreIt->second; + if ( std::fpclassify( p[i] * N * D ) == FP_SUBNORMAL ) + { + m_err << "Error: encountered subnormal value: p[i] * N * D= " << p[i] * N * D + << " with p[i]= " << p[i] << " N= " << N << " D= " << D << '\n'; + m_err.flush(); + exit(1); + } else { + m_gradient[name] += p[i] * N * D; + if ( maintainUpdateSet ) + { + m_updateSet.insert(name); + } + } + } + } + + m_xBleu += xBleu; +} + + +void ExpectedBleuOptimizer::InitSGD(const std::vector<float>& sparseScalingFactor) +{ + const size_t nFeatures = sparseScalingFactor.size(); + memcpy(&m_previousSparseScalingFactor.at(0), &sparseScalingFactor.at(0), nFeatures); + m_gradient.resize(nFeatures); +} + + +float ExpectedBleuOptimizer::UpdateSGD(std::vector<float>& sparseScalingFactor, + size_t batchSize, + bool useUpdateSet) +{ + + float xBleu = m_xBleu / batchSize; + + // update sparse scaling factors + + if (useUpdateSet) { + + for (std::set<size_t>::const_iterator it = m_updateSet.begin(); it != m_updateSet.end(); ++it) + { + size_t name = *it; + UpdateSingleScalingFactorSGD(name, sparseScalingFactor, batchSize); + } + + m_updateSet.clear(); + + } else { + + for (size_t name=0; name<sparseScalingFactor.size(); ++name) + { + UpdateSingleScalingFactorSGD(name, sparseScalingFactor, batchSize); + } + + } + + m_xBleu = 0; + m_gradient.clear(); + return xBleu; +} + + +void ExpectedBleuOptimizer::UpdateSingleScalingFactorSGD(size_t name, + std::vector<float>& sparseScalingFactor, + size_t batchSize) +{ + // regularization + if ( m_regularizationParameter != 0 ) + { + m_gradient[name] = m_gradient[name] / m_xBleu - m_regularizationParameter * 2 * sparseScalingFactor[name]; + } else { + // need to normalize by dividing by batchSize + m_gradient[name] /= batchSize; + } + + // the actual update + sparseScalingFactor[name] += m_learningRate * m_gradient[name]; + + // discard scaling factors below a threshold + if ( fabs(sparseScalingFactor[name]) < m_floorAbsScalingFactor ) + { + sparseScalingFactor[name] = 0; + } +} + + +void ExpectedBleuOptimizer::InitRPROP(const std::vector<float>& sparseScalingFactor) +{ + const size_t nFeatures = sparseScalingFactor.size(); + m_previousSparseScalingFactor.resize(nFeatures); + memcpy(&m_previousSparseScalingFactor.at(0), &sparseScalingFactor.at(0), nFeatures); + m_previousGradient.resize(nFeatures); + m_gradient.resize(nFeatures); + m_stepSize.resize(nFeatures, m_initialStepSize); +} + + +float ExpectedBleuOptimizer::UpdateRPROP(std::vector<float>& sparseScalingFactor, + const size_t batchSize) +{ + + float xBleu = m_xBleu / batchSize; + + // update sparse scaling factors + + for (size_t name=0; name<sparseScalingFactor.size(); ++name) + { + // Sum of gradients. All we need is the sign. Don't need to normalize by dividing by batchSize. + + // regularization + if ( m_regularizationParameter != 0 ) + { + m_gradient[name] = m_gradient[name] / m_xBleu - m_regularizationParameter * 2 * sparseScalingFactor[name]; + } + + // step size + int sign = Sign(m_gradient[name]) * Sign(m_previousGradient[name]); + if (sign > 0) { + m_stepSize[name] *= m_increaseRate; + } else if (sign < 0) { + m_stepSize[name] *= m_decreaseRate; + } + if (m_stepSize[name] < m_minStepSize) { + m_stepSize[name] = m_minStepSize; + } + if (m_stepSize[name] > m_maxStepSize) { + m_stepSize[name] = m_maxStepSize; + } + + // the actual update + + m_previousGradient[name] = m_gradient[name]; + if (sign >= 0) { + if (m_gradient[name] > 0) { + m_previousSparseScalingFactor[name] = sparseScalingFactor[name]; + sparseScalingFactor[name] += m_stepSize[name]; + } else if (m_gradient[name] < 0) { + m_previousSparseScalingFactor[name] = sparseScalingFactor[name]; + sparseScalingFactor[name] -= m_stepSize[name]; + } + } else { + sparseScalingFactor[name] = m_previousSparseScalingFactor[name]; + // m_previousGradient[name] = 0; + } + + // discard scaling factors below a threshold + if ( fabs(sparseScalingFactor[name]) < m_floorAbsScalingFactor ) + { + sparseScalingFactor[name] = 0; + } + } + + m_xBleu = 0; + m_gradient.clear(); + return xBleu; +} + + +} + diff --git a/contrib/expected-bleu-training/ExpectedBleuOptimizer.h b/contrib/expected-bleu-training/ExpectedBleuOptimizer.h new file mode 100644 index 000000000..438f7d161 --- /dev/null +++ b/contrib/expected-bleu-training/ExpectedBleuOptimizer.h @@ -0,0 +1,117 @@ +/* + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + + +#pragma once + +#include <vector> +#include <set> +#include <boost/unordered_map.hpp> +#include "util/file_stream.hh" + + +namespace ExpectedBleuTraining +{ + +class ExpectedBleuOptimizer +{ +public: + + ExpectedBleuOptimizer(util::FileStream& err, + float learningRate=1, + float initialStepSize=0.001, + float decreaseRate=0.5, + float increaseRate=1.2, + float minStepSize=1e-7, + float maxStepSize=1, + float floorAbsScalingFactor=0, + float regularizationParameter=0) + : m_err(err) + , m_learningRate(learningRate) + , m_initialStepSize(initialStepSize) + , m_decreaseRate(decreaseRate) + , m_increaseRate(increaseRate) + , m_minStepSize(minStepSize) + , m_maxStepSize(maxStepSize) + , m_floorAbsScalingFactor(floorAbsScalingFactor) + , m_regularizationParameter(regularizationParameter) + , m_xBleu(0) + { } + + void AddTrainingInstance(const size_t nBestSizeCount, + const std::vector<float>& sBleu, + const std::vector<double>& overallScoreUntransformed, + const std::vector< boost::unordered_map<size_t, float> > &sparseScore, + bool maintainUpdateSet = false); + + void InitSGD(const std::vector<float>& sparseScalingFactor); + + float UpdateSGD(std::vector<float>& sparseScalingFactor, + size_t batchSize, + bool useUpdateSet = false); + + void InitRPROP(const std::vector<float>& sparseScalingFactor); + + float UpdateRPROP(std::vector<float>& sparseScalingFactor, + const size_t batchSize); + +protected: + + util::FileStream& m_err; + + // for SGD + const float m_learningRate; + + // for RPROP + const float m_initialStepSize; + const float m_decreaseRate; + const float m_increaseRate; + const float m_minStepSize; + const float m_maxStepSize; + + std::vector<float> m_previousSparseScalingFactor; + std::vector<float> m_previousGradient; + std::vector<float> m_gradient; + std::vector<float> m_stepSize; + + // other + const float m_floorAbsScalingFactor; + const float m_regularizationParameter; + + double m_xBleu; + + std::set<size_t> m_updateSet; + + + void UpdateSingleScalingFactorSGD(size_t name, + std::vector<float>& sparseScalingFactor, + size_t batchSize); + + + inline int Sign(double x) + { + if (x > 0) return 1; + if (x < 0) return -1; + return 0; + } +}; + +} + + diff --git a/contrib/expected-bleu-training/Jamfile b/contrib/expected-bleu-training/Jamfile new file mode 100644 index 000000000..3d660650e --- /dev/null +++ b/contrib/expected-bleu-training/Jamfile @@ -0,0 +1,2 @@ +exe prepare-expected-bleu-training : PrepareExpectedBleuTraining.cpp ../../util//kenutil ; +exe train-expected-bleu : TrainExpectedBleu.cpp ExpectedBleuOptimizer.cpp ../../util//kenutil ; diff --git a/contrib/expected-bleu-training/PrepareExpectedBleuTraining.cpp b/contrib/expected-bleu-training/PrepareExpectedBleuTraining.cpp new file mode 100644 index 000000000..6873494b5 --- /dev/null +++ b/contrib/expected-bleu-training/PrepareExpectedBleuTraining.cpp @@ -0,0 +1,222 @@ +/* + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + + +#include <vector> +#include <string> +#include <sstream> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/unordered_map.hpp> +#include <boost/unordered_set.hpp> +#include <boost/program_options.hpp> +#include "util/file_stream.hh" +#include "util/file.hh" +#include "util/file_piece.hh" +#include "util/string_piece.hh" +#include "util/tokenize_piece.hh" + +namespace po = boost::program_options; + + +int main(int argc, char **argv) +{ + util::FileStream err(2); + + std::string filenameNBestListIn, filenameFeatureNamesOut, filenameIgnoreFeatureNames; + size_t maxNBestSize; + + try { + + po::options_description descr("Usage"); + descr.add_options() + ("help,h", "produce help message") + ("n-best-list,n", po::value<std::string>(&filenameNBestListIn)->required(), + "input n-best list file") + ("write-feature-names-file,f", po::value<std::string>(&filenameFeatureNamesOut)->required(), + "output file for mapping between feature names and indices") + ("ignore-features-file,i", po::value<std::string>(&filenameIgnoreFeatureNames)->required(), + "input file containing list of feature names to be ignored") + ("n-best-size-limit,l", po::value<size_t>(&maxNBestSize)->default_value(100), + "limit of n-best list entries to be considered") + ; + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, descr), vm); + + if (vm.count("help")) { + std::ostringstream os; + os << descr; + std::cout << os.str() << '\n'; + exit(0); + } + + po::notify(vm); + + } catch(std::exception& e) { + + err << "Error: " << e.what() << '\n'; + err.flush(); + exit(1); + } + + util::FilePiece ifsNBest(filenameNBestListIn.c_str()); + util::FilePiece ifsIgnoreFeatureNames(filenameIgnoreFeatureNames.c_str()); + util::scoped_fd fdFeatureNames(util::CreateOrThrow(filenameFeatureNamesOut.c_str())); + util::FileStream ofsFeatureNames(fdFeatureNames.get()); + util::FileStream ofsNBest(1); + + boost::unordered_set<std::string> ignoreFeatureNames; + StringPiece line; + + while ( ifsIgnoreFeatureNames.ReadLineOrEOF(line) ) + { + if ( !line.empty() ) { + util::TokenIter<util::AnyCharacter> item(line, " \t="); + if ( item != item.end() ) + { + ignoreFeatureNames.insert(item->as_string()); + } + err << "ignoring " << *item << '\n'; + } + } + + size_t maxFeatureNamesIdx = 0; + boost::unordered_map<std::string, size_t> featureNames; + + size_t sentenceIndex = 0; + size_t nBestSizeCount = 0; + size_t globalIndex = 0; + + while ( ifsNBest.ReadLineOrEOF(line) ) + { + util::TokenIter<util::MultiCharacter> item(line, " ||| "); + + if ( item == item.end() ) + { + err << "Error: flawed content in " << filenameNBestListIn << '\n'; + exit(1); + } + + size_t sentenceIndexCurrent = atol( item->as_string().c_str() ); + + if ( sentenceIndex != sentenceIndexCurrent ) + { + nBestSizeCount = 0; + sentenceIndex = sentenceIndexCurrent; + } + + if ( nBestSizeCount < maxNBestSize ) + { + // process n-best list entry + + StringPiece scores; + StringPiece decoderScore; + for (size_t nItem=1; nItem<=3; ++nItem) + { + if ( ++item == item.end() ) { + err << "Error: flawed content in " << filenameNBestListIn << '\n'; + exit(1); + } + if (nItem == 2) { + scores = *item; + } + if (nItem == 3) { + decoderScore = *item; + } + } + + ofsNBest << sentenceIndex << ' ' + << decoderScore; + + util::TokenIter<util::SingleCharacter> token(scores, ' '); + std::string featureNameCurrent("ERROR"); + std::string featureNameCurrentBase("ERROR"); + bool ignore = false; + int scoreComponentIndex = 0; + + while ( token != token.end() ) + { + if ( token->ends_with("=") ) + { + scoreComponentIndex = 0; + featureNameCurrent = token->substr(0,token->size()-1).as_string(); + size_t idx = featureNameCurrent.find_first_of('_'); + if ( idx == StringPiece::npos ) { + featureNameCurrentBase = featureNameCurrent; + } else { + featureNameCurrentBase = featureNameCurrent.substr(0,idx+1); + } + ignore = false; + if ( ignoreFeatureNames.find(featureNameCurrentBase) != ignoreFeatureNames.end() ) + { + ignore = true; + } else { + if ( (featureNameCurrent.compare(featureNameCurrentBase)) && + (ignoreFeatureNames.find(featureNameCurrent) != ignoreFeatureNames.end()) ) + { + ignore = true; + } + } + } + else + { + if ( !ignore ) + { + float featureValueCurrent = atof( token->as_string().c_str() );; + if ( scoreComponentIndex > 0 ) + { + std::ostringstream oss; + oss << scoreComponentIndex; + featureNameCurrent.append("+"); + } + if ( featureValueCurrent != 0 ) + { + boost::unordered_map<std::string, size_t>::iterator featureName = featureNames.find(featureNameCurrent); + + if ( featureName == featureNames.end() ) + { + std::pair< boost::unordered_map<std::string, size_t>::iterator, bool> inserted = + featureNames.insert( std::make_pair(featureNameCurrent, maxFeatureNamesIdx) ); + ++maxFeatureNamesIdx; + featureName = inserted.first; + } + + ofsNBest << ' ' << featureName->second // feature name index + << ' ' << *token; // feature value + } + ++scoreComponentIndex; + } + } + ++token; + } + ofsNBest << '\n'; + ++nBestSizeCount; + } + ++globalIndex; + } + + ofsFeatureNames << maxFeatureNamesIdx << '\n'; + for (boost::unordered_map<std::string, size_t>::const_iterator featureNamesIt=featureNames.begin(); + featureNamesIt!=featureNames.end(); ++featureNamesIt) + { + ofsFeatureNames << featureNamesIt->second << ' ' << featureNamesIt->first << '\n'; + } + +} + diff --git a/contrib/expected-bleu-training/TrainExpectedBleu.cpp b/contrib/expected-bleu-training/TrainExpectedBleu.cpp new file mode 100644 index 000000000..1c6c06321 --- /dev/null +++ b/contrib/expected-bleu-training/TrainExpectedBleu.cpp @@ -0,0 +1,379 @@ +/* + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + + +#include "ExpectedBleuOptimizer.h" +#include "util/file_stream.hh" +#include "util/file_piece.hh" +#include "util/string_piece.hh" +#include "util/tokenize_piece.hh" + +#include <sstream> +#include <boost/program_options.hpp> + +using namespace ExpectedBleuTraining; +namespace po = boost::program_options; + + +int main(int argc, char **argv) { + + util::FileStream out(1); + util::FileStream err(2); + + size_t maxNBestSize; + size_t iterationLimit; + std::string filenameSBleu, filenameNBestList, filenameFeatureNames, filenameInitialWeights; + + bool ignoreDecoderScore; + + float learningRate; + float initialStepSize; + float decreaseRate; + float increaseRate; + float minStepSize; + float maxStepSize; + float floorAbsScalingFactor; + float regularizationParameter; + bool printZeroWeights; + bool miniBatches; + std::string optimizerTypeStr; + size_t optimizerType = 0; +#define EXPECTED_BLEU_OPTIMIZER_TYPE_RPROP 1 +#define EXPECTED_BLEU_OPTIMIZER_TYPE_SGD 2 + + try { + + po::options_description descr("Usage"); + descr.add_options() + ("help,h", "produce help message") + ("n-best-size-limit,l", po::value<size_t>(&maxNBestSize)->default_value(100), + "limit of n-best list entries to be considered for training") + ("iterations,i", po::value<size_t>(&iterationLimit)->default_value(50), + "number of training iterations") + ("sbleu-file,b", po::value<std::string>(&filenameSBleu)->required(), + "file containing sentence-level BLEU scores for all n-best list entries") + ("prepared-n-best-list,n", po::value<std::string>(&filenameNBestList)->required(), + "input n-best list file, in prepared format for expected BLEU training") + ("feature-name-file,f", po::value<std::string>(&filenameFeatureNames)->required(), + "file containing mapping between feature names and indices") + ("initial-weights-file,w", po::value<std::string>(&filenameInitialWeights)->default_value(""), + "file containing start values for scaling factors (optional)") + ("ignore-decoder-score", boost::program_options::value<bool>(&ignoreDecoderScore)->default_value(0), + "exclude decoder score from computation of posterior probability") + ("regularization", boost::program_options::value<float>(®ularizationParameter)->default_value(0), // e.g. 1e-5 + "regularization parameter; suggested value range: [1e-8,1e-5]") + ("learning-rate", boost::program_options::value<float>(&learningRate)->default_value(1), + "learning rate for the SGD optimizer") + ("floor", boost::program_options::value<float>(&floorAbsScalingFactor)->default_value(0), // e.g. 1e-7 + "set scaling factor to 0 if below this absolute value after update") + ("initial-step-size", boost::program_options::value<float>(&initialStepSize)->default_value(0.001), // TODO: try 0.01 and 0.1 + "initial step size for the RPROP optimizer") + ("decrease-rate", boost::program_options::value<float>(&decreaseRate)->default_value(0.5), + "decrease rate for the RPROP optimizer") + ("increase-rate", boost::program_options::value<float>(&increaseRate)->default_value(1.2), + "increase rate for the RPROP optimizer") + ("min-step-size", boost::program_options::value<float>(&minStepSize)->default_value(1e-7), + "minimum step size for the RPROP optimizer") + ("max-step-size", boost::program_options::value<float>(&maxStepSize)->default_value(1), + "maximum step size for the RPROP optimizer") + ("print-zero-weights", boost::program_options::value<bool>(&printZeroWeights)->default_value(0), + "output scaling factors even if they are trained to 0") + ("optimizer", po::value<std::string>(&optimizerTypeStr)->default_value("RPROP"), + "optimizer type used for training (known algorithms: RPROP, SGD)") + ("mini-batches", boost::program_options::value<bool>(&miniBatches)->default_value(0), + "update after every single sentence (SGD only)") + ; + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, descr), vm); + + if (vm.count("help")) { + std::ostringstream os; + os << descr; + out << os.str() << '\n'; + out.flush(); + exit(0); + } + + po::notify(vm); + + } catch(std::exception& e) { + + err << "Error: " << e.what() << '\n'; + err.flush(); + exit(1); + } + + if ( !optimizerTypeStr.compare("rprop") || !optimizerTypeStr.compare("RPROP") ) { + optimizerType = EXPECTED_BLEU_OPTIMIZER_TYPE_RPROP; + } else if ( !optimizerTypeStr.compare("sgd") || !optimizerTypeStr.compare("SGD") ) { + optimizerType = EXPECTED_BLEU_OPTIMIZER_TYPE_SGD; + } else { + err << "Error: unknown optimizer type: \"" << optimizerTypeStr << "\" (known optimizers: rprop, sgd) " << '\n'; + err.flush(); + exit(1); + } + + + + util::FilePiece ifsFeatureNames(filenameFeatureNames.c_str()); + + StringPiece lineFeatureName; + if ( !ifsFeatureNames.ReadLineOrEOF(lineFeatureName) ) + { + err << "Error: flawed content in " << filenameFeatureNames << '\n'; + err.flush(); + exit(1); + } + size_t maxFeatureNamesIdx = atol( lineFeatureName.as_string().c_str() ); + + std::vector<std::string> featureNames(maxFeatureNamesIdx); + boost::unordered_map<std::string, size_t> featureIndexes; + for (size_t i=0; i<maxFeatureNamesIdx; ++i) + { + if ( !ifsFeatureNames.ReadLineOrEOF(lineFeatureName) ) { + err << "Error: flawed content in " << filenameFeatureNames << '\n'; + err.flush(); + exit(1); + } + util::TokenIter<util::SingleCharacter> token(lineFeatureName, ' '); + size_t featureIndexCurrent = atol( token->as_string().c_str() ); + token++; + featureNames[featureIndexCurrent] = token->as_string(); + featureIndexes[token->as_string()] = featureIndexCurrent; + } + + + std::vector<float> sparseScalingFactor(maxFeatureNamesIdx); + std::vector< boost::unordered_map<size_t, float> > sparseScore(maxNBestSize); + + // read initial weights, if any given + + if ( filenameInitialWeights.length() != 0 ) + { + util::FilePiece ifsInitialWeights(filenameInitialWeights.c_str()); + + StringPiece lineInitialWeight; + if ( !ifsInitialWeights.ReadLineOrEOF(lineInitialWeight) ) { + err << "Error: flawed content in " << filenameInitialWeights << '\n'; + err.flush(); + exit(1); + } + do { + util::TokenIter<util::SingleCharacter> token(lineInitialWeight, ' '); + boost::unordered_map<std::string, size_t>::const_iterator found = featureIndexes.find(token->as_string()); + if ( found == featureIndexes.end() ) { + err << "Error: flawed content in " << filenameInitialWeights << " (unkown feature name \"" << token->as_string() << "\")" << '\n'; + err.flush(); + exit(1); + } + token++; + sparseScalingFactor[found->second] = atof( token->as_string().c_str() ); + } while ( ifsInitialWeights.ReadLineOrEOF(lineInitialWeight) ); + } + + // train + + ExpectedBleuOptimizer optimizer(err, + learningRate, + initialStepSize, + decreaseRate, + increaseRate, + minStepSize, + maxStepSize, + floorAbsScalingFactor, + regularizationParameter); + + if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_RPROP ) + { + optimizer.InitRPROP(sparseScalingFactor); + } else if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_SGD ) { + optimizer.InitRPROP(sparseScalingFactor); + } else { + err << "Error: unknown optimizer type" << '\n'; + err.flush(); + exit(1); + } + + for (size_t nIteration=1; nIteration<=iterationLimit; ++nIteration) + { + util::FilePiece ifsSBleu(filenameSBleu.c_str()); + util::FilePiece ifsNBest(filenameNBestList.c_str()); + + out << "### ITERATION " << nIteration << '\n' << '\n'; + + size_t sentenceIndex = 0; + size_t batchSize = 0; + size_t nBestSizeCount = 0; + size_t globalIndex = 0; + StringPiece lineNBest; + std::vector<double> overallScoreUntransformed; + std::vector<float> sBleu; + float xBleu = 0; + // double expPrecisionCorrection = 0.0; + + while ( ifsNBest.ReadLineOrEOF(lineNBest) ) + { + + util::TokenIter<util::SingleCharacter> token(lineNBest, ' '); + + if ( token == token.end() ) + { + err << "Error: flawed content in " << filenameNBestList << '\n'; + err.flush(); + exit(1); + } + + size_t sentenceIndexCurrent = atol( token->as_string().c_str() ); + token++; + + if ( sentenceIndex != sentenceIndexCurrent ) + { + + if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_RPROP ) + { + optimizer.AddTrainingInstance( nBestSizeCount, sBleu, overallScoreUntransformed, sparseScore ); + } else if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_SGD ) { + optimizer.AddTrainingInstance( nBestSizeCount, sBleu, overallScoreUntransformed, sparseScore, miniBatches ); + + if ( miniBatches ) { + xBleu += optimizer.UpdateSGD( sparseScalingFactor, 1 ); + // out << "ITERATION " << nIteration << " SENTENCE " << sentenceIndex << " XBLEUSUM= " << xBleu << '\n'; + // for (size_t i=0; i<sparseScalingFactor.size(); ++i) + // { + // if ( (sparseScalingFactor[i] != 0) || printZeroWeights ) + // { + // out << "ITERATION " << nIteration << " WEIGHT " << featureNames[i] << " " << sparseScalingFactor[i] << '\n'; + // } + // } + // out << '\n'; + // out.flush(); + } + } else { + err << "Error: unknown optimizer type" << '\n'; + err.flush(); + exit(1); + } + + for (size_t i=0; i<nBestSizeCount; ++i) { + sparseScore[i].clear(); + } + nBestSizeCount = 0; + overallScoreUntransformed.clear(); + sBleu.clear(); + sentenceIndex = sentenceIndexCurrent; + ++batchSize; + } + + StringPiece lineSBleu; + if ( !ifsSBleu.ReadLineOrEOF(lineSBleu) ) + { + err << "Error: insufficient number of lines in " << filenameSBleu << '\n'; + err.flush(); + exit(1); + } + + if ( nBestSizeCount < maxNBestSize ) + { + // retrieve sBLEU + + float sBleuCurrent = atof( lineSBleu.as_string().c_str() ); + sBleu.push_back(sBleuCurrent); + + // process n-best list entry + + if ( token == token.end() ) + { + err << "Error: flawed content in " << filenameNBestList << '\n'; + err.flush(); + exit(1); + } + double scoreCurrent = 0; + if ( !ignoreDecoderScore ) + { + scoreCurrent = atof( token->as_string().c_str() ); // decoder score + } + token++; + + // if ( nBestSizeCount == 0 ) // best translation (first n-best list entry for the current sentence / a new mini-batch) + // { + // expPrecisionCorrection = std::floor ( scoreCurrent ); // decoder score of first-best + // } + + while (token != token.end()) + { + size_t featureNameCurrent = atol( token->as_string().c_str() ); + token++; + float featureValueCurrent = atof( token->as_string().c_str() ); + sparseScore[nBestSizeCount].insert(std::make_pair(featureNameCurrent, featureValueCurrent)); + scoreCurrent += sparseScalingFactor[featureNameCurrent] * featureValueCurrent; + token++; + } + + // overallScoreUntransformed.push_back( std::exp(scoreCurrent - expPrecisionCorrection) ); + overallScoreUntransformed.push_back( std::exp(scoreCurrent) ); + + ++nBestSizeCount; + } + ++globalIndex; + } + + if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_RPROP ) + { + optimizer.AddTrainingInstance( nBestSizeCount, sBleu, overallScoreUntransformed, sparseScore ); // last sentence in the corpus + xBleu = optimizer.UpdateRPROP( sparseScalingFactor, batchSize ); + out << "xBLEU= " << xBleu << '\n'; + } else if ( optimizerType == EXPECTED_BLEU_OPTIMIZER_TYPE_SGD ) { + optimizer.AddTrainingInstance( nBestSizeCount, sBleu, overallScoreUntransformed, sparseScore, miniBatches ); // last sentence in the corpus + if ( miniBatches ) { + xBleu += optimizer.UpdateSGD( sparseScalingFactor, 1 ); + xBleu /= batchSize; + } else { + xBleu = optimizer.UpdateSGD( sparseScalingFactor, batchSize ); + } + out << "xBLEU= " << xBleu << '\n'; + } else { + err << "Error: unknown optimizer type" << '\n'; + err.flush(); + exit(1); + } + + for (size_t i=0; i<nBestSizeCount; ++i) { + sparseScore[i].clear(); + } + nBestSizeCount = 0; + overallScoreUntransformed.clear(); + sBleu.clear(); + + out << '\n'; + + for (size_t i=0; i<sparseScalingFactor.size(); ++i) + { + if ( (sparseScalingFactor[i] != 0) || printZeroWeights ) + { + out << "ITERATION " << nIteration << " WEIGHT " << featureNames[i] << " " << sparseScalingFactor[i] << '\n'; + } + } + + out << '\n'; + out.flush(); + } + +} + diff --git a/contrib/makemteval/makemteval.py b/contrib/makemteval/makemteval.py index 11b428ac6..7dc37c1d5 100644 --- a/contrib/makemteval/makemteval.py +++ b/contrib/makemteval/makemteval.py @@ -221,7 +221,7 @@ Copyright © 2010-2014 Precision Translation Tools Co., Ltd. This module is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
+the Free Software Foundation, either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project index 8b1013bf1..ce7912954 100644 --- a/contrib/other-builds/moses/.project +++ b/contrib/other-builds/moses/.project @@ -96,6 +96,16 @@ <locationURI>PARENT-3-PROJECT_LOC/moses/BitmapContainer.h</locationURI> </link> <link> + <name>Bitmaps.cpp</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/Bitmaps.cpp</locationURI> + </link> + <link> + <name>Bitmaps.h</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/Bitmaps.h</locationURI> + </link> + <link> <name>BookkeepingOptions.cpp</name> <type>1</type> <locationURI>PARENT-3-PROJECT_LOC/moses/parameters/BookkeepingOptions.cpp</locationURI> @@ -1256,6 +1266,16 @@ <locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.h</locationURI> </link> <link> + <name>FF/DeleteRules.cpp</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/FF/DeleteRules.cpp</locationURI> + </link> + <link> + <name>FF/DeleteRules.h</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/FF/DeleteRules.h</locationURI> + </link> + <link> <name>FF/DistortionScoreProducer.cpp</name> <type>1</type> <locationURI>PARENT-3-PROJECT_LOC/moses/FF/DistortionScoreProducer.cpp</locationURI> @@ -2171,6 +2191,16 @@ <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h</locationURI> </link> <link> + <name>TranslationModel/PhraseDictionaryGroup.cpp</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryGroup.cpp</locationURI> + </link> + <link> + <name>TranslationModel/PhraseDictionaryGroup.h</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryGroup.h</locationURI> + </link> + <link> <name>TranslationModel/PhraseDictionaryMemory.cpp</name> <type>1</type> <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemory.cpp</locationURI> @@ -3126,6 +3156,11 @@ <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/StringVector.h</locationURI> </link> <link> + <name>TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp</name> + <type>1</type> + <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp</locationURI> + </link> + <link> <name>TranslationModel/CompactPT/TargetPhraseCollectionCache.h</name> <type>1</type> <locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h</locationURI> diff --git a/contrib/sigtest-filter/Makefile b/contrib/sigtest-filter/Makefile index 55772929a..c7bbd1c58 100644 --- a/contrib/sigtest-filter/Makefile +++ b/contrib/sigtest-filter/Makefile @@ -1,5 +1,5 @@ SALMDIR=/Users/hieuhoang/workspace/salm -FLAVOR?=o64 +FLAVOR?=o32 INC=-I$(SALMDIR)/Src/Shared -I$(SALMDIR)/Src/SuffixArrayApplications -I$(SALMDIR)/Src/SuffixArrayApplications/SuffixArraySearch OBJS=$(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArrayApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArraySearchApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_String.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_IDVocabulary.$(FLAVOR) diff --git a/defer/PhraseDictionaryInterpolated.cpp b/defer/PhraseDictionaryInterpolated.cpp index 83abf73ba..892e5f98f 100644 --- a/defer/PhraseDictionaryInterpolated.cpp +++ b/defer/PhraseDictionaryInterpolated.cpp @@ -116,7 +116,7 @@ typedef boost::unordered_set<TargetPhrase*,PhrasePtrHasher,PhrasePtrComparator> PhraseSet; -const TargetPhraseCollection* +TargetPhraseCollection::shared_ptr PhraseDictionaryInterpolated::GetTargetPhraseCollection(const Phrase& src) const { @@ -125,7 +125,7 @@ PhraseDictionaryInterpolated::GetTargetPhraseCollection(const Phrase& src) const PhraseSet allPhrases; vector<PhraseSet> phrasesByTable(m_dictionaries.size()); for (size_t i = 0; i < m_dictionaries.size(); ++i) { - const TargetPhraseCollection* phrases = m_dictionaries[i]->GetTargetPhraseCollection(src); + TargetPhraseCollection::shared_ptr phrases = m_dictionaries[i]->GetTargetPhraseCollection(src); if (phrases) { for (TargetPhraseCollection::const_iterator j = phrases->begin(); j != phrases->end(); ++j) { diff --git a/defer/PhraseDictionaryInterpolated.h b/defer/PhraseDictionaryInterpolated.h index aee1de6fa..dea00d892 100644 --- a/defer/PhraseDictionaryInterpolated.h +++ b/defer/PhraseDictionaryInterpolated.h @@ -52,7 +52,7 @@ public: , const LMList &languageModels , float weightWP); - virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const; + virtual TargetPhraseCollection::shared_ptr GetTargetPhraseCollection(const Phrase& src) const; virtual void InitializeForInput(ttasksptr const& ttask); virtual ChartRuleLookupManager *CreateRuleLookupManager( const InputType &, @@ -65,7 +65,7 @@ private: typedef boost::shared_ptr<PhraseDictionaryTreeAdaptor> DictionaryHandle; std::vector<DictionaryHandle> m_dictionaries; std::vector<std::vector<float> > m_weights; //feature x table - mutable TargetPhraseCollection* m_targetPhrases; + mutable TargetPhraseCollection::shared_ptr m_targetPhrases; std::vector<float> m_weightT; size_t m_tableLimit; const LMList* m_languageModels; diff --git a/lm/binary_format.cc b/lm/binary_format.cc index 2b34a778a..802943f57 100644 --- a/lm/binary_format.cc +++ b/lm/binary_format.cc @@ -169,8 +169,7 @@ void *BinaryFormat::SetupJustVocab(std::size_t memory_size, uint8_t order) { vocab_size_ = memory_size; if (!write_mmap_) { header_size_ = 0; - util::MapAnonymous(memory_size, memory_vocab_); - util::AdviseHugePages(memory_vocab_.get(), memory_size); + util::HugeMalloc(memory_size, true, memory_vocab_); return reinterpret_cast<uint8_t*>(memory_vocab_.get()); } header_size_ = TotalHeaderSize(order); @@ -181,16 +180,16 @@ void *BinaryFormat::SetupJustVocab(std::size_t memory_size, uint8_t order) { switch (write_method_) { case Config::WRITE_MMAP: mapping_.reset(util::MapZeroedWrite(file_.get(), total), total, util::scoped_memory::MMAP_ALLOCATED); + util::AdviseHugePages(vocab_base, total); vocab_base = mapping_.get(); break; case Config::WRITE_AFTER: util::ResizeOrThrow(file_.get(), 0); - util::MapAnonymous(total, memory_vocab_); + util::HugeMalloc(total, true, memory_vocab_); vocab_base = memory_vocab_.get(); break; } strncpy(reinterpret_cast<char*>(vocab_base), kMagicIncomplete, header_size_); - util::AdviseHugePages(vocab_base, total); return reinterpret_cast<uint8_t*>(vocab_base) + header_size_; } @@ -200,7 +199,7 @@ void *BinaryFormat::GrowForSearch(std::size_t memory_size, std::size_t vocab_pad std::size_t new_size = header_size_ + vocab_size_ + vocab_pad_ + memory_size; vocab_string_offset_ = new_size; if (!write_mmap_ || write_method_ == Config::WRITE_AFTER) { - util::MapAnonymous(memory_size, memory_search_); + util::HugeMalloc(memory_size, true, memory_search_); assert(header_size_ == 0 || write_mmap_); vocab_base = reinterpret_cast<uint8_t*>(memory_vocab_.get()) + header_size_; util::AdviseHugePages(memory_search_.get(), memory_size); diff --git a/lm/builder/corpus_count.cc b/lm/builder/corpus_count.cc index 04815d805..0414c2261 100644 --- a/lm/builder/corpus_count.cc +++ b/lm/builder/corpus_count.cc @@ -5,7 +5,7 @@ #include "lm/lm_exception.hh" #include "lm/vocab.hh" #include "lm/word_index.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/file_piece.hh" #include "util/murmur_hash.hh" diff --git a/lm/builder/debug_print.hh b/lm/builder/debug_print.hh index 193a6892c..4b9f306d8 100644 --- a/lm/builder/debug_print.hh +++ b/lm/builder/debug_print.hh @@ -4,21 +4,21 @@ #include "lm/builder/payload.hh" #include "lm/common/print.hh" #include "lm/common/ngram_stream.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include <boost/lexical_cast.hpp> namespace lm { namespace builder { // Not defined, only specialized. -template <class T> void PrintPayload(util::FakeOFStream &to, const BuildingPayload &payload); -template <> inline void PrintPayload<uint64_t>(util::FakeOFStream &to, const BuildingPayload &payload) { +template <class T> void PrintPayload(util::FileStream &to, const BuildingPayload &payload); +template <> inline void PrintPayload<uint64_t>(util::FileStream &to, const BuildingPayload &payload) { to << payload.count; } -template <> inline void PrintPayload<Uninterpolated>(util::FakeOFStream &to, const BuildingPayload &payload) { +template <> inline void PrintPayload<Uninterpolated>(util::FileStream &to, const BuildingPayload &payload) { to << log10(payload.uninterp.prob) << ' ' << log10(payload.uninterp.gamma); } -template <> inline void PrintPayload<ProbBackoff>(util::FakeOFStream &to, const BuildingPayload &payload) { +template <> inline void PrintPayload<ProbBackoff>(util::FileStream &to, const BuildingPayload &payload) { to << payload.complete.prob << ' ' << payload.complete.backoff; } @@ -36,7 +36,7 @@ template <class V> class Print { void Run(const util::stream::ChainPositions &chains) { util::scoped_fd fd(to_); - util::FakeOFStream out(to_); + util::FileStream out(to_); NGramStreams<BuildingPayload> streams(chains); for (NGramStream<BuildingPayload> *s = streams.begin(); s != streams.end(); ++s) { DumpStream(*s, out); @@ -45,13 +45,13 @@ template <class V> class Print { void Run(const util::stream::ChainPosition &position) { util::scoped_fd fd(to_); - util::FakeOFStream out(to_); + util::FileStream out(to_); NGramStream<BuildingPayload> stream(position); DumpStream(stream, out); } private: - void DumpStream(NGramStream<BuildingPayload> &stream, util::FakeOFStream &to) { + void DumpStream(NGramStream<BuildingPayload> &stream, util::FileStream &to) { for (; stream; ++stream) { PrintPayload<V>(to, stream->Value()); for (const WordIndex *w = stream->begin(); w != stream->end(); ++w) { diff --git a/lm/builder/dump_counts_main.cc b/lm/builder/dump_counts_main.cc index a4c9478b6..26078d0e7 100644 --- a/lm/builder/dump_counts_main.cc +++ b/lm/builder/dump_counts_main.cc @@ -30,7 +30,7 @@ int main(int argc, char *argv[]) { UTIL_THROW_IF(*i >= vocab.Size(), util::Exception, "Vocab ID " << *i << " is larger than the vocab file's maximum of " << vocab.Size() << ". Are you sure you have the right order and vocab file for these counts?"); std::cout << vocab.Lookup(*i) << ' '; } - // TODO don't use std::cout because it is slow. Add fast uint64_t printing support to FakeOFStream. + // TODO don't use std::cout because it is slow. Add fast uint64_t printing support to FileStream. std::cout << *reinterpret_cast<const uint64_t*>(words + order) << '\n'; } } diff --git a/lm/builder/interpolate.cc b/lm/builder/interpolate.cc index 7dcb3623c..a62ef43d2 100644 --- a/lm/builder/interpolate.cc +++ b/lm/builder/interpolate.cc @@ -12,7 +12,6 @@ #include <iostream> #include <cassert> #include <cmath> -#include <iostream> namespace lm { namespace builder { namespace { diff --git a/lm/builder/output.cc b/lm/builder/output.cc index c92283ac6..604fa22e6 100644 --- a/lm/builder/output.cc +++ b/lm/builder/output.cc @@ -2,7 +2,7 @@ #include "lm/common/model_buffer.hh" #include "lm/common/print.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/stream/multi_stream.hh" #include <iostream> @@ -41,7 +41,7 @@ void Output::Apply(HookType hook_type, util::stream::Chains &chains) { void PrintHook::Sink(const HeaderInfo &info, int vocab_file, util::stream::Chains &chains) { if (verbose_header_) { - util::FakeOFStream out(file_.get(), 50); + util::FileStream out(file_.get(), 50); out << "# Input file: " << info.input_file << '\n'; out << "# Token count: " << info.token_count << '\n'; out << "# Smoothing: Modified Kneser-Ney" << '\n'; diff --git a/lm/common/model_buffer.cc b/lm/common/model_buffer.cc index 431d4ae4c..ae9b08c4f 100644 --- a/lm/common/model_buffer.cc +++ b/lm/common/model_buffer.cc @@ -1,6 +1,6 @@ #include "lm/common/model_buffer.hh" #include "util/exception.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/file_piece.hh" #include "util/stream/io.hh" @@ -68,7 +68,7 @@ void ModelBuffer::Sink(util::stream::Chains &chains, const std::vector<uint64_t> } if (keep_buffer_) { util::scoped_fd metadata(util::CreateOrThrow((file_base_ + ".kenlm_intermediate").c_str())); - util::FakeOFStream meta(metadata.get(), 200); + util::FileStream meta(metadata.get(), 200); meta << kMetadataHeader << "\nCounts"; for (std::vector<uint64_t>::const_iterator i = counts_.begin(); i != counts_.end(); ++i) { meta << ' ' << *i; diff --git a/lm/common/print.cc b/lm/common/print.cc index cd2a80260..518b62f51 100644 --- a/lm/common/print.cc +++ b/lm/common/print.cc @@ -1,7 +1,7 @@ #include "lm/common/print.hh" #include "lm/common/ngram_stream.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/mmap.hh" #include "util/scoped.hh" @@ -24,7 +24,7 @@ VocabReconstitute::VocabReconstitute(int fd) { } namespace { -template <class Payload> void PrintLead(const VocabReconstitute &vocab, ProxyStream<Payload> &stream, util::FakeOFStream &out) { +template <class Payload> void PrintLead(const VocabReconstitute &vocab, ProxyStream<Payload> &stream, util::FileStream &out) { out << stream->Value().prob << '\t' << vocab.Lookup(*stream->begin()); for (const WordIndex *i = stream->begin() + 1; i != stream->end(); ++i) { out << ' ' << vocab.Lookup(*i); @@ -34,7 +34,7 @@ template <class Payload> void PrintLead(const VocabReconstitute &vocab, ProxyStr void PrintARPA::Run(const util::stream::ChainPositions &positions) { VocabReconstitute vocab(vocab_fd_); - util::FakeOFStream out(out_fd_); + util::FileStream out(out_fd_); out << "\\data\\\n"; for (size_t i = 0; i < positions.size(); ++i) { out << "ngram " << (i+1) << '=' << counts_[i] << '\n'; diff --git a/lm/filter/arpa_io.cc b/lm/filter/arpa_io.cc index 92c821aaf..cf2721ed5 100644 --- a/lm/filter/arpa_io.cc +++ b/lm/filter/arpa_io.cc @@ -1,5 +1,6 @@ #include "lm/filter/arpa_io.hh" #include "util/file_piece.hh" +#include "util/string_stream.hh" #include <iostream> #include <ostream> @@ -22,14 +23,8 @@ ARPAInputException::ARPAInputException(const StringPiece &message, const StringP ARPAInputException::~ARPAInputException() throw() {} -ARPAOutputException::ARPAOutputException(const char *message, const std::string &file_name) throw() { - *this << message << " in file " << file_name; -} - -ARPAOutputException::~ARPAOutputException() throw() {} - // Seeking is the responsibility of the caller. -void WriteCounts(std::ostream &out, const std::vector<uint64_t> &number) { +template <class Stream> void WriteCounts(Stream &out, const std::vector<uint64_t> &number) { out << "\n\\data\\\n"; for (unsigned int i = 0; i < number.size(); ++i) { out << "ngram " << i+1 << "=" << number[i] << '\n'; @@ -38,9 +33,9 @@ void WriteCounts(std::ostream &out, const std::vector<uint64_t> &number) { } size_t SizeNeededForCounts(const std::vector<uint64_t> &number) { - std::ostringstream buf; - WriteCounts(buf, number); - return buf.tellp(); + util::StringStream stream; + WriteCounts(stream, number); + return stream.str().size(); } bool IsEntirelyWhiteSpace(const StringPiece &line) { @@ -50,44 +45,21 @@ bool IsEntirelyWhiteSpace(const StringPiece &line) { return true; } -ARPAOutput::ARPAOutput(const char *name, size_t buffer_size) : file_name_(name), buffer_(new char[buffer_size]) { - try { - file_.exceptions(std::ostream::eofbit | std::ostream::failbit | std::ostream::badbit); - if (!file_.rdbuf()->pubsetbuf(buffer_.get(), buffer_size)) { - std::cerr << "Warning: could not enlarge buffer for " << name << std::endl; - buffer_.reset(); - } - file_.open(name, std::ios::out | std::ios::binary); - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Opening", file_name_); - } -} +ARPAOutput::ARPAOutput(const char *name, size_t buffer_size) + : file_backing_(util::CreateOrThrow(name)), file_(file_backing_.get(), buffer_size) {} void ARPAOutput::ReserveForCounts(std::streampos reserve) { - try { - for (std::streampos i = 0; i < reserve; i += std::streampos(1)) { - file_ << '\n'; - } - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Writing blanks to reserve space for counts to ", file_name_); + for (std::streampos i = 0; i < reserve; i += std::streampos(1)) { + file_ << '\n'; } } void ARPAOutput::BeginLength(unsigned int length) { - fast_counter_ = 0; - try { - file_ << '\\' << length << "-grams:" << '\n'; - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Writing n-gram header to ", file_name_); - } + file_ << '\\' << length << "-grams:" << '\n'; } void ARPAOutput::EndLength(unsigned int length) { - try { - file_ << '\n'; - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Writing blank at end of count list to ", file_name_); - } + file_ << '\n'; if (length > counts_.size()) { counts_.resize(length); } @@ -95,14 +67,10 @@ void ARPAOutput::EndLength(unsigned int length) { } void ARPAOutput::Finish() { - try { - file_ << "\\end\\\n"; - file_.seekp(0); - WriteCounts(file_, counts_); - file_ << std::flush; - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Finishing including writing counts at beginning to ", file_name_); - } + file_ << "\\end\\\n"; + file_.seekp(0); + WriteCounts(file_, counts_); + file_.flush(); } } // namespace lm diff --git a/lm/filter/arpa_io.hh b/lm/filter/arpa_io.hh index 0f6c8be75..7489270db 100644 --- a/lm/filter/arpa_io.hh +++ b/lm/filter/arpa_io.hh @@ -4,6 +4,7 @@ */ #include "lm/read_arpa.hh" #include "util/exception.hh" +#include "util/file_stream.hh" #include "util/string_piece.hh" #include "util/tokenize_piece.hh" @@ -28,17 +29,6 @@ class ARPAInputException : public util::Exception { virtual ~ARPAInputException() throw(); }; -class ARPAOutputException : public util::ErrnoException { - public: - ARPAOutputException(const char *prefix, const std::string &file_name) throw(); - virtual ~ARPAOutputException() throw(); - - const std::string &File() const throw() { return file_name_; } - - private: - const std::string file_name_; -}; - // Handling for the counts of n-grams at the beginning of ARPA files. size_t SizeNeededForCounts(const std::vector<uint64_t> &number); @@ -55,11 +45,7 @@ class ARPAOutput : boost::noncopyable { void BeginLength(unsigned int length); void AddNGram(const StringPiece &line) { - try { - file_ << line << '\n'; - } catch (const std::ios_base::failure &f) { - throw ARPAOutputException("Writing an n-gram", file_name_); - } + file_ << line << '\n'; ++fast_counter_; } @@ -76,9 +62,8 @@ class ARPAOutput : boost::noncopyable { void Finish(); private: - const std::string file_name_; - boost::scoped_array<char> buffer_; - std::fstream file_; + util::scoped_fd file_backing_; + util::FileStream file_; size_t fast_counter_; std::vector<uint64_t> counts_; }; diff --git a/lm/filter/count_io.hh b/lm/filter/count_io.hh index 02eb78baa..1af6676c4 100644 --- a/lm/filter/count_io.hh +++ b/lm/filter/count_io.hh @@ -5,7 +5,7 @@ #include <iostream> #include <string> -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/file_piece.hh" @@ -28,7 +28,7 @@ class CountOutput : boost::noncopyable { } private: - util::FakeOFStream file_; + util::FileStream file_; }; class CountBatch { diff --git a/lm/filter/phrase_table_vocab_main.cc b/lm/filter/phrase_table_vocab_main.cc index e8a8d0265..9ffa35f93 100644 --- a/lm/filter/phrase_table_vocab_main.cc +++ b/lm/filter/phrase_table_vocab_main.cc @@ -1,4 +1,4 @@ -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file_piece.hh" #include "util/murmur_hash.hh" #include "util/pool.hh" @@ -68,7 +68,7 @@ class TargetWords { } void Print() const { - util::FakeOFStream out(1); + util::FileStream out(1); for (std::vector<boost::unordered_set<const char *> >::const_iterator i = vocab_.begin(); i != vocab_.end(); ++i) { for (boost::unordered_set<const char *>::const_iterator j = i->begin(); j != i->end(); ++j) { out << *j << ' '; diff --git a/lm/kenlm_benchmark_main.cc b/lm/kenlm_benchmark_main.cc index 1704ec6c9..c9ee16525 100644 --- a/lm/kenlm_benchmark_main.cc +++ b/lm/kenlm_benchmark_main.cc @@ -1,5 +1,5 @@ #include "lm/model.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/file_piece.hh" #include "util/usage.hh" @@ -10,7 +10,7 @@ namespace { template <class Model, class Width> void ConvertToBytes(const Model &model, int fd_in) { util::FilePiece in(fd_in); - util::FakeOFStream out(1); + util::FileStream out(1); Width width; StringPiece word; const Width end_sentence = (Width)model.GetVocabulary().EndSentence(); @@ -30,10 +30,19 @@ template <class Model, class Width> void QueryFromBytes(const Model &model, int const lm::ngram::State *next_state = begin_state; Width kEOS = model.GetVocabulary().EndSentence(); Width buf[4096]; - float sum = 0.0; + + uint64_t completed = 0; + double loaded = util::CPUTime(); + + std::cout << "CPU_to_load: " << loaded << std::endl; + + // Numerical precision: batch sums. + double total = 0.0; while (std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf))) { + float sum = 0.0; UTIL_THROW_IF2(got % sizeof(Width), "File size not a multiple of vocab id size " << sizeof(Width)); got /= sizeof(Width); + completed += got; // Do even stuff first. const Width *even_end = buf + (got & ~1); // Alternating states @@ -49,8 +58,13 @@ template <class Model, class Width> void QueryFromBytes(const Model &model, int sum += model.FullScore(*next_state, *i, state[2]).prob; next_state = (*i++ == kEOS) ? begin_state : &state[2]; } + total += sum; } - std::cout << "Sum is " << sum << std::endl; + double after = util::CPUTime(); + std::cerr << "Probability sum is " << total << std::endl; + std::cout << "Queries: " << completed << std::endl; + std::cout << "CPU_excluding_load: " << (after - loaded) << "\nCPU_per_query: " << ((after - loaded) / static_cast<double>(completed)) << std::endl; + std::cout << "RSSMax: " << util::RSSMax() << std::endl; } template <class Model, class Width> void DispatchFunction(const Model &model, bool query) { @@ -62,7 +76,10 @@ template <class Model, class Width> void DispatchFunction(const Model &model, bo } template <class Model> void DispatchWidth(const char *file, bool query) { - Model model(file); + lm::ngram::Config config; + config.load_method = util::READ; + std::cerr << "Using load_method = READ." << std::endl; + Model model(file, config); lm::WordIndex bound = model.GetVocabulary().Bound(); if (bound <= 256) { DispatchFunction<Model, uint8_t>(model, query); @@ -116,11 +133,10 @@ int main(int argc, char *argv[]) { << argv[0] << " vocab $model <$text >$text.vocab\n" << "#Ensure files are in RAM.\n" << "cat $text.vocab $model >/dev/null\n" - << "#Timed query against the model, including loading.\n" - << "time " << argv[0] << " query $model <$text.vocab\n"; + << "#Timed query against the model.\n" + << argv[0] << " query $model <$text.vocab\n"; return 1; } Dispatch(argv[2], !strcmp(argv[1], "query")); - util::PrintUsage(std::cerr); return 0; } diff --git a/lm/ngram_query.hh b/lm/ngram_query.hh index b19c5aa4f..4430841c9 100644 --- a/lm/ngram_query.hh +++ b/lm/ngram_query.hh @@ -3,7 +3,7 @@ #include "lm/enumerate_vocab.hh" #include "lm/model.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file_piece.hh" #include "util/usage.hh" @@ -42,7 +42,7 @@ class QueryPrinter { } private: - util::FakeOFStream out_; + util::FileStream out_; bool print_word_; bool print_line_; bool print_summary_; diff --git a/lm/vocab.cc b/lm/vocab.cc index 5696e60b3..3d83e0452 100644 --- a/lm/vocab.cc +++ b/lm/vocab.cc @@ -6,7 +6,7 @@ #include "lm/config.hh" #include "lm/weights.hh" #include "util/exception.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/joint_sort.hh" #include "util/murmur_hash.hh" @@ -182,7 +182,7 @@ void SortedVocabulary::ComputeRenumbering(WordIndex types, int from_words, int t std::sort(entries.begin(), entries.end()); // Write out new vocab file. { - util::FakeOFStream out(to_words); + util::FileStream out(to_words); out << "<unk>" << '\0'; for (std::vector<RenumberEntry>::const_iterator i = entries.begin(); i != entries.end(); ++i) { out << i->str << '\0'; diff --git a/lm/vocab.hh b/lm/vocab.hh index b42566f23..59740e853 100644 --- a/lm/vocab.hh +++ b/lm/vocab.hh @@ -4,7 +4,7 @@ #include "lm/enumerate_vocab.hh" #include "lm/lm_exception.hh" #include "lm/virtual_interface.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/murmur_hash.hh" #include "util/pool.hh" #include "util/probing_hash_table.hh" @@ -44,7 +44,7 @@ class ImmediateWriteWordsWrapper : public EnumerateVocab { private: EnumerateVocab *inner_; - util::FakeOFStream stream_; + util::FileStream stream_; }; // When the binary size isn't known yet. @@ -225,7 +225,7 @@ class WriteUniqueWords { } private: - util::FakeOFStream word_list_; + util::FileStream word_list_; }; class NoOpUniqueWords { diff --git a/mert/TER/alignmentStruct.cpp b/mert/TER/alignmentStruct.cpp index 849e3390f..73ce83bbf 100644 --- a/mert/TER/alignmentStruct.cpp +++ b/mert/TER/alignmentStruct.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT @@ -141,4 +141,4 @@ void alignmentStruct::set(alignmentStruct l_alignmentStruct) // } -}
\ No newline at end of file +} diff --git a/mert/TER/alignmentStruct.h b/mert/TER/alignmentStruct.h index 379af9382..b5183bbed 100644 --- a/mert/TER/alignmentStruct.h +++ b/mert/TER/alignmentStruct.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/bestShiftStruct.cpp b/mert/TER/bestShiftStruct.cpp index c8d44e3e3..98a60b2fa 100644 --- a/mert/TER/bestShiftStruct.cpp +++ b/mert/TER/bestShiftStruct.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/bestShiftStruct.h b/mert/TER/bestShiftStruct.h index 85b65979b..3a480b09b 100644 --- a/mert/TER/bestShiftStruct.h +++ b/mert/TER/bestShiftStruct.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMap.cpp b/mert/TER/hashMap.cpp index 71166db30..0f069ef53 100644 --- a/mert/TER/hashMap.cpp +++ b/mert/TER/hashMap.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMap.h b/mert/TER/hashMap.h index bc947fab1..a19942d6d 100644 --- a/mert/TER/hashMap.h +++ b/mert/TER/hashMap.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMapInfos.cpp b/mert/TER/hashMapInfos.cpp index 39b354c7a..f890b7ecc 100644 --- a/mert/TER/hashMapInfos.cpp +++ b/mert/TER/hashMapInfos.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMapInfos.h b/mert/TER/hashMapInfos.h index ae7cbbda2..d418e8ba4 100644 --- a/mert/TER/hashMapInfos.h +++ b/mert/TER/hashMapInfos.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMapStringInfos.cpp b/mert/TER/hashMapStringInfos.cpp index 29f138ea6..21267fbe7 100644 --- a/mert/TER/hashMapStringInfos.cpp +++ b/mert/TER/hashMapStringInfos.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/hashMapStringInfos.h b/mert/TER/hashMapStringInfos.h index 5413d97fa..283d75ef8 100644 --- a/mert/TER/hashMapStringInfos.h +++ b/mert/TER/hashMapStringInfos.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/infosHasher.cpp b/mert/TER/infosHasher.cpp index 201003b54..cd725a177 100644 --- a/mert/TER/infosHasher.cpp +++ b/mert/TER/infosHasher.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/infosHasher.h b/mert/TER/infosHasher.h index b824d18eb..5a6ec05a6 100644 --- a/mert/TER/infosHasher.h +++ b/mert/TER/infosHasher.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/stringHasher.cpp b/mert/TER/stringHasher.cpp index 6519f9819..02c44b20d 100644 --- a/mert/TER/stringHasher.cpp +++ b/mert/TER/stringHasher.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/stringHasher.h b/mert/TER/stringHasher.h index a57d2c65b..0b5c94b45 100644 --- a/mert/TER/stringHasher.h +++ b/mert/TER/stringHasher.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/stringInfosHasher.cpp b/mert/TER/stringInfosHasher.cpp index f5870fda1..4264571bb 100644 --- a/mert/TER/stringInfosHasher.cpp +++ b/mert/TER/stringInfosHasher.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/stringInfosHasher.h b/mert/TER/stringInfosHasher.h index 140bab096..b876596af 100644 --- a/mert/TER/stringInfosHasher.h +++ b/mert/TER/stringInfosHasher.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT @@ -49,4 +49,4 @@ public: } -#endif
\ No newline at end of file +#endif diff --git a/mert/TER/terAlignment.cpp b/mert/TER/terAlignment.cpp index ced587d60..3d320cb42 100644 --- a/mert/TER/terAlignment.cpp +++ b/mert/TER/terAlignment.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT @@ -234,4 +234,4 @@ string terAlignment::printAllShifts() return to_return.str(); } -}
\ No newline at end of file +} diff --git a/mert/TER/terAlignment.h b/mert/TER/terAlignment.h index 0e98c8f9a..4444ec896 100644 --- a/mert/TER/terAlignment.h +++ b/mert/TER/terAlignment.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/terShift.cpp b/mert/TER/terShift.cpp index f1c302f92..e1ea33801 100644 --- a/mert/TER/terShift.cpp +++ b/mert/TER/terShift.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT @@ -165,4 +165,4 @@ int terShift::size() // } -}
\ No newline at end of file +} diff --git a/mert/TER/terShift.h b/mert/TER/terShift.h index 1f9746e60..70d64ccb6 100644 --- a/mert/TER/terShift.h +++ b/mert/TER/terShift.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/tercalc.cpp b/mert/TER/tercalc.cpp index 110f70a6d..0d342e2db 100644 --- a/mert/TER/tercalc.cpp +++ b/mert/TER/tercalc.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/tercalc.h b/mert/TER/tercalc.h index 24fb10c4c..271f271b6 100644 --- a/mert/TER/tercalc.h +++ b/mert/TER/tercalc.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/tools.cpp b/mert/TER/tools.cpp index 1c0d68a5b..1634e481e 100644 --- a/mert/TER/tools.cpp +++ b/mert/TER/tools.cpp @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/mert/TER/tools.h b/mert/TER/tools.h index 20a0cf6b0..8b4f9dade 100644 --- a/mert/TER/tools.h +++ b/mert/TER/tools.h @@ -6,7 +6,7 @@ Contact: christophe.servan@lium.univ-lemans.fr The tercpp tool and library are free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the licence, or +the Free Software Foundation, either version 2.1 of the licence, or (at your option) any later version. This program and library are distributed in the hope that it will be useful, but WITHOUT diff --git a/misc/pmoses/pmoses.cc b/misc/pmoses/pmoses.cc index 1bbfb6b1d..77ee568d4 100644 --- a/misc/pmoses/pmoses.cc +++ b/misc/pmoses/pmoses.cc @@ -1,21 +1,20 @@ -/* - * Copyright (C) 2009 Felipe Sánchez-Martínez - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA - * 02111-1307, USA. - */ +/*********************************************************************** +Copyright (C) 2009 Felipe Sánchez-Martínez + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ #include <string> #include <vector> diff --git a/mmt b/mmt new file mode 160000 +Subproject 1edfd04a1b62f93dffd3d0cbf724d6b255c8a61 diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 7de3206fd..5ee99e2db 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main. **/ #include "moses/ExportInterface.h" +#include "util/string_stream.hh" /** main function of the command line version of the decoder **/ int main(int argc, char** argv) diff --git a/moses/BitmapContainer.cpp b/moses/BitmapContainer.cpp index 40ec74153..3f0e5f99a 100644 --- a/moses/BitmapContainer.cpp +++ b/moses/BitmapContainer.cpp @@ -110,13 +110,13 @@ public: BackwardsEdge::BackwardsEdge(const BitmapContainer &prevBitmapContainer , BitmapContainer &parent , const TranslationOptionList &translations - , const SquareMatrix &futureScore, + , const SquareMatrix &futureScores, const InputType& itype) : m_initialized(false) , m_prevBitmapContainer(prevBitmapContainer) , m_parent(parent) , m_translations(translations) - , m_futurescore(futureScore) + , m_futureScores(futureScores) , m_seenPosition() { @@ -195,6 +195,10 @@ BackwardsEdge::Initialize() return; } + const WordsBitmap &bm = m_hypotheses[0]->GetWordsBitmap(); + const WordsRange &newRange = m_translations.Get(0)->GetSourceWordsRange(); + m_futureScore = m_futureScores.CalcFutureScore2(bm, newRange.GetStartPos(), newRange.GetEndPos()); + Hypothesis *expanded = CreateHypothesis(*m_hypotheses[0], *m_translations.Get(0)); m_parent.Enqueue(0, 0, expanded, this); SetSeenPosition(0, 0); @@ -207,11 +211,12 @@ Hypothesis *BackwardsEdge::CreateHypothesis(const Hypothesis &hypothesis, const IFVERBOSE(2) { hypothesis.GetManager().GetSentenceStats().StartTimeBuildHyp(); } - Hypothesis *newHypo = hypothesis.CreateNext(transOpt); // TODO FIXME This is absolutely broken - don't pass null here + const WordsBitmap &bitmap = m_parent.GetWordsBitmap(); + Hypothesis *newHypo = new Hypothesis(hypothesis, transOpt, bitmap); IFVERBOSE(2) { hypothesis.GetManager().GetSentenceStats().StopTimeBuildHyp(); } - newHypo->EvaluateWhenApplied(m_futurescore); + newHypo->EvaluateWhenApplied(m_futureScore); return newHypo; } @@ -273,9 +278,11 @@ BackwardsEdge::PushSuccessors(const size_t x, const size_t y) //////////////////////////////////////////////////////////////////////////////// BitmapContainer::BitmapContainer(const WordsBitmap &bitmap - , HypothesisStackCubePruning &stack) + , HypothesisStackCubePruning &stack + , bool deterministic) : m_bitmap(bitmap) , m_stack(stack) + , m_deterministic(deterministic) , m_numStackInsertions(0) { m_hypotheses = HypothesisSet(); @@ -291,7 +298,7 @@ BitmapContainer::~BitmapContainer() HypothesisQueueItem *item = m_queue.top(); m_queue.pop(); - FREEHYPO( item->GetHypothesis() ); + delete item->GetHypothesis(); delete item; } @@ -309,10 +316,13 @@ BitmapContainer::Enqueue(int hypothesis_pos , Hypothesis *hypothesis , BackwardsEdge *edge) { + // Only supply target phrase if running deterministic search mode + const TargetPhrase *target_phrase = m_deterministic ? &(hypothesis->GetCurrTargetPhrase()) : NULL; HypothesisQueueItem *item = new HypothesisQueueItem(hypothesis_pos , translation_pos , hypothesis - , edge); + , edge + , target_phrase); IFVERBOSE(2) { item->GetHypothesis()->GetManager().GetSentenceStats().StartTimeManageCubes(); } @@ -356,13 +366,6 @@ BitmapContainer::Empty() const return m_queue.empty(); } - -const WordsBitmap& -BitmapContainer::GetWordsBitmap() -{ - return m_bitmap; -} - const HypothesisSet& BitmapContainer::GetHypotheses() const { diff --git a/moses/BitmapContainer.h b/moses/BitmapContainer.h index 500059081..6c500ff71 100644 --- a/moses/BitmapContainer.h +++ b/moses/BitmapContainer.h @@ -61,6 +61,7 @@ private: size_t m_hypothesis_pos, m_translation_pos; Hypothesis *m_hypothesis; BackwardsEdge *m_edge; + boost::shared_ptr<TargetPhrase> m_target_phrase; HypothesisQueueItem(); @@ -68,11 +69,15 @@ public: HypothesisQueueItem(const size_t hypothesis_pos , const size_t translation_pos , Hypothesis *hypothesis - , BackwardsEdge *edge) + , BackwardsEdge *edge + , const TargetPhrase *target_phrase = NULL) : m_hypothesis_pos(hypothesis_pos) , m_translation_pos(translation_pos) , m_hypothesis(hypothesis) , m_edge(edge) { + if (target_phrase != NULL) { + m_target_phrase.reset(new TargetPhrase(*target_phrase)); + } } ~HypothesisQueueItem() { @@ -93,6 +98,10 @@ public: BackwardsEdge *GetBackwardsEdge() { return m_edge; } + + boost::shared_ptr<TargetPhrase> GetTargetPhrase() { + return m_target_phrase; + } }; //! Allows comparison of two HypothesisQueueItem objects by the corresponding scores. @@ -103,20 +112,20 @@ public: float scoreA = itemA->GetHypothesis()->GetTotalScore(); float scoreB = itemB->GetHypothesis()->GetTotalScore(); - return (scoreA < scoreB); - - /* - { - return true; + if (scoreA < scoreB) { + return true; + } else if (scoreA > scoreB) { + return false; + } else { + // Equal scores: break ties by comparing target phrases (if they exist) + boost::shared_ptr<TargetPhrase> phrA = itemA->GetTargetPhrase(); + boost::shared_ptr<TargetPhrase> phrB = itemB->GetTargetPhrase(); + if (!phrA || !phrB) { + // Fallback: scoreA < scoreB == false, non-deterministic sort + return false; + } + return (phrA->Compare(*phrB) < 0); } - else if (scoreA < scoreB) - { - return false; - } - else - { - return itemA < itemB; - }*/ } }; @@ -134,18 +143,6 @@ public: float scoreB = hypoB->GetTotalScore(); return (scoreA > scoreB); - /* - { - return true; - } - else if (scoreA < scoreB) - { - return false; - } - else - { - return hypoA < hypoB; - }*/ } }; @@ -164,7 +161,8 @@ private: const BitmapContainer &m_prevBitmapContainer; BitmapContainer &m_parent; const TranslationOptionList &m_translations; - const SquareMatrix &m_futurescore; + const SquareMatrix &m_futureScores; + float m_futureScore; std::vector< const Hypothesis* > m_hypotheses; boost::unordered_set< int > m_seenPosition; @@ -183,7 +181,7 @@ public: BackwardsEdge(const BitmapContainer &prevBitmapContainer , BitmapContainer &parent , const TranslationOptionList &translations - , const SquareMatrix &futureScore, + , const SquareMatrix &futureScores, const InputType& source); ~BackwardsEdge(); @@ -204,19 +202,21 @@ public: class BitmapContainer { private: - WordsBitmap m_bitmap; + const WordsBitmap m_bitmap; HypothesisStackCubePruning &m_stack; HypothesisSet m_hypotheses; BackwardsEdgeSet m_edges; HypothesisQueue m_queue; size_t m_numStackInsertions; + bool m_deterministic; // We always require a corresponding bitmap to be supplied. BitmapContainer(); BitmapContainer(const BitmapContainer &); public: BitmapContainer(const WordsBitmap &bitmap - , HypothesisStackCubePruning &stack); + , HypothesisStackCubePruning &stack + , bool deterministic_sort = false); // The destructor will also delete all the edges that are // connected to this BitmapContainer. @@ -228,7 +228,10 @@ public: size_t Size(); bool Empty() const; - const WordsBitmap &GetWordsBitmap(); + const WordsBitmap &GetWordsBitmap() const { + return m_bitmap; + } + const HypothesisSet &GetHypotheses() const; size_t GetHypothesesSize() const; const BackwardsEdgeSet &GetBackwardsEdges(); diff --git a/moses/Bitmaps.cpp b/moses/Bitmaps.cpp new file mode 100644 index 000000000..bb9bd20bd --- /dev/null +++ b/moses/Bitmaps.cpp @@ -0,0 +1,59 @@ +#include <boost/foreach.hpp> +#include "Bitmaps.h" +#include "Util.h" + +using namespace std; + +namespace Moses +{ +Bitmaps::Bitmaps(size_t inputSize, const std::vector<bool> &initSourceCompleted) +{ + m_initBitmap = new WordsBitmap(inputSize, initSourceCompleted); + m_coll[m_initBitmap]; +} + +Bitmaps::~Bitmaps() +{ + BOOST_FOREACH (const Coll::value_type& myPair, m_coll) { + const WordsBitmap *bm = myPair.first; + delete bm; + } +} + +const WordsBitmap &Bitmaps::GetNextBitmap(const WordsBitmap &bm, const WordsRange &range) +{ + WordsBitmap *newBM = new WordsBitmap(bm); + newBM->SetValue(range, true); + + Coll::const_iterator iter = m_coll.find(newBM); + if (iter == m_coll.end()) { + m_coll[newBM] = NextBitmaps(); + return *newBM; + } else { + delete newBM; + return *iter->first; + } +} + +const WordsBitmap &Bitmaps::GetBitmap(const WordsBitmap &bm, const WordsRange &range) +{ + Coll::iterator iter = m_coll.find(&bm); + assert(iter != m_coll.end()); + + const WordsBitmap *newBM; + NextBitmaps &next = iter->second; + NextBitmaps::const_iterator iterNext = next.find(range); + if (iterNext == next.end()) { + // not seen the link yet. + newBM = &GetNextBitmap(bm, range); + next[range] = newBM; + } else { + // link exist + //std::cerr << "link exists" << endl; + newBM = iterNext->second; + } + return *newBM; +} + +} + diff --git a/moses/Bitmaps.h b/moses/Bitmaps.h new file mode 100644 index 000000000..4f2337e39 --- /dev/null +++ b/moses/Bitmaps.h @@ -0,0 +1,32 @@ +#pragma once + +#include <boost/unordered_set.hpp> +#include <boost/unordered_map.hpp> +#include <set> +#include "WordsBitmap.h" +#include "Util.h" + +namespace Moses +{ + +class Bitmaps +{ + typedef boost::unordered_map<WordsRange, const WordsBitmap*> NextBitmaps; + typedef boost::unordered_map<const WordsBitmap*, NextBitmaps, UnorderedComparer<WordsBitmap>, UnorderedComparer<WordsBitmap> > Coll; + //typedef std::set<const WordsBitmap*, OrderedComparer<WordsBitmap> > Coll; + Coll m_coll; + const WordsBitmap *m_initBitmap; + + const WordsBitmap &GetNextBitmap(const WordsBitmap &bm, const WordsRange &range); +public: + Bitmaps(size_t inputSize, const std::vector<bool> &initSourceCompleted); + virtual ~Bitmaps(); + + const WordsBitmap &GetInitialBitmap() const { + return *m_initBitmap; + } + const WordsBitmap &GetBitmap(const WordsBitmap &bm, const WordsRange &range); + +}; + +} diff --git a/moses/ChartCell.h b/moses/ChartCell.h index d9213d5e1..348bfd336 100644 --- a/moses/ChartCell.h +++ b/moses/ChartCell.h @@ -120,11 +120,6 @@ public: void OutputSizes(std::ostream &out) const; size_t GetSize() const; - //! transitive comparison used for adding objects into set - inline bool operator<(const ChartCell &compare) const { - return m_coverage < compare.m_coverage; - } - void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const; }; diff --git a/moses/ChartCellLabel.h b/moses/ChartCellLabel.h index c67d985b2..18fa8b850 100644 --- a/moses/ChartCellLabel.h +++ b/moses/ChartCellLabel.h @@ -78,15 +78,6 @@ public: return m_bestScore; } - bool operator<(const ChartCellLabel &other) const { - // m_coverage and m_label uniquely identify a ChartCellLabel, so don't - // need to compare m_stack. - if (m_coverage == other.m_coverage) { - return m_label < other.m_label; - } - return m_coverage < other.m_coverage; - } - private: const WordsRange &m_coverage; const Word &m_label; diff --git a/moses/ChartHypothesis.cpp b/moses/ChartHypothesis.cpp index 806222029..5f92b19a9 100644 --- a/moses/ChartHypothesis.cpp +++ b/moses/ChartHypothesis.cpp @@ -37,10 +37,6 @@ using namespace std; namespace Moses { -#ifdef USE_HYPO_POOL -ObjectPool<ChartHypothesis> ChartHypothesis::s_objectPool("ChartHypothesis", 300000); -#endif - /** Create a hypothesis from a rule * \param transOpt wrapper around the rule * \param item @todo dunno @@ -93,7 +89,7 @@ ChartHypothesis::~ChartHypothesis() ChartArcList::iterator iter; for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) { ChartHypothesis *hypo = *iter; - Delete(hypo); + delete hypo; } m_arcList->clear(); @@ -181,32 +177,6 @@ void ChartHypothesis::GetOutputPhrase(size_t leftRightMost, size_t numWords, Phr } } -/** check, if two hypothesis can be recombined. - this is actually a sorting function that allows us to - keep an ordered list of hypotheses. This makes recombination - much quicker. Returns one of 3 possible values: - -1 = this < compare - +1 = this > compare - 0 = this ==compare - \param compare the other hypo to compare to -*/ -int ChartHypothesis::RecombineCompare(const ChartHypothesis &compare) const -{ - int comp = 0; - - for (unsigned i = 0; i < m_ffStates.size(); ++i) { - if (m_ffStates[i] == NULL || compare.m_ffStates[i] == NULL) - comp = m_ffStates[i] - compare.m_ffStates[i]; - else - comp = m_ffStates[i]->Compare(*compare.m_ffStates[i]); - - if (comp != 0) - return comp; - } - - return 0; -} - /** calculate total score */ void ChartHypothesis::EvaluateWhenApplied() { @@ -304,7 +274,7 @@ void ChartHypothesis::CleanupArcList() ChartArcList::iterator iter; for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) { ChartHypothesis *arc = *iter; - ChartHypothesis::Delete(arc); + delete arc; } m_arcList->erase(m_arcList->begin() + nBestSize , m_arcList->end()); @@ -325,6 +295,33 @@ void ChartHypothesis::SetWinningHypo(const ChartHypothesis *hypo) m_winningHypo = hypo; } +size_t ChartHypothesis::hash() const +{ + size_t seed; + + // states + for (size_t i = 0; i < m_ffStates.size(); ++i) { + const FFState *state = m_ffStates[i]; + size_t hash = state->hash(); + boost::hash_combine(seed, hash); + } + return seed; + +} + +bool ChartHypothesis::operator==(const ChartHypothesis& other) const +{ + // states + for (size_t i = 0; i < m_ffStates.size(); ++i) { + const FFState &thisState = *m_ffStates[i]; + const FFState &otherState = *other.m_ffStates[i]; + if (thisState != otherState) { + return false; + } + } + return true; +} + TO_STRING_BODY(ChartHypothesis) // friend diff --git a/moses/ChartHypothesis.h b/moses/ChartHypothesis.h index 25216c04c..776fcd61c 100644 --- a/moses/ChartHypothesis.h +++ b/moses/ChartHypothesis.h @@ -49,9 +49,6 @@ class ChartHypothesis // friend class ChartKBestExtractor; protected: -#ifdef USE_HYPO_POOL - static ObjectPool<ChartHypothesis> s_objectPool; -#endif boost::shared_ptr<ChartTranslationOption> m_transOpt; @@ -81,23 +78,6 @@ protected: ChartHypothesis(const ChartHypothesis ©); public: -#ifdef USE_HYPO_POOL - void *operator new(size_t /* num_bytes */) { - void *ptr = s_objectPool.getPtr(); - return ptr; - } - - //! delete \param hypo. Works with object pool too - static void Delete(ChartHypothesis *hypo) { - s_objectPool.freeObject(hypo); - } -#else - //! delete \param hypo. Works with object pool too - static void Delete(ChartHypothesis *hypo) { - delete hypo; - } -#endif - ChartHypothesis(const ChartTranslationOptions &, const RuleCubeItem &item, ChartManager &manager); @@ -146,8 +126,6 @@ public: // leftRightMost: 1=left, 2=right void GetOutputPhrase(size_t leftRightMost, size_t numWords, Phrase &outPhrase) const; - int RecombineCompare(const ChartHypothesis &compare) const; - void EvaluateWhenApplied(); void AddArc(ChartHypothesis *loserHypo); @@ -214,6 +192,10 @@ public: return m_winningHypo; } + // for unordered_set in stack + size_t hash() const; + bool operator==(const ChartHypothesis& other) const; + TO_STRING(); }; // class ChartHypothesis diff --git a/moses/ChartHypothesisCollection.cpp b/moses/ChartHypothesisCollection.cpp index 42717c261..068194287 100644 --- a/moses/ChartHypothesisCollection.cpp +++ b/moses/ChartHypothesisCollection.cpp @@ -48,7 +48,7 @@ ChartHypothesisCollection::~ChartHypothesisCollection() HCType::iterator iter; for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) { ChartHypothesis *hypo = *iter; - ChartHypothesis::Delete(hypo); + delete hypo; } //RemoveAllInColl(m_hypos); } @@ -65,7 +65,7 @@ bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManage if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) { manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, -inf score" << std::endl); - ChartHypothesis::Delete(hypo); + delete hypo; return false; } @@ -73,7 +73,7 @@ bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManage // really bad score. don't bother adding hypo into collection manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, too bad for stack" << std::endl); - ChartHypothesis::Delete(hypo); + delete hypo; return false; } @@ -118,7 +118,7 @@ bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManage if (m_nBestIsEnabled) { hypoExisting->AddArc(hypo); } else { - ChartHypothesis::Delete(hypo); + delete hypo; } return false; } @@ -167,22 +167,8 @@ void ChartHypothesisCollection::Detach(const HCType::iterator &iter) void ChartHypothesisCollection::Remove(const HCType::iterator &iter) { ChartHypothesis *h = *iter; - - /* - stringstream strme(""); - strme << h->GetOutputPhrase(); - string toFind = "the goal of gene scientists is "; - size_t pos = toFind.find(strme.str()); - - if (pos == 0) - { - cerr << pos << " " << strme.str() << *h << endl; - cerr << *this << endl; - } - */ - Detach(iter); - ChartHypothesis::Delete(h); + delete h; } /** prune number of hypo to a particular number of hypos, specified by m_maxHypoStackSize, according to score diff --git a/moses/ChartHypothesisCollection.h b/moses/ChartHypothesisCollection.h index b2464e151..169e81f19 100644 --- a/moses/ChartHypothesisCollection.h +++ b/moses/ChartHypothesisCollection.h @@ -39,23 +39,6 @@ public: } }; -/** functor to compare (chart) hypotheses by feature function states. - * If 2 hypos are equal, according to this functor, then they can be recombined. - */ -class ChartHypothesisRecombinationOrderer -{ -public: - bool operator()(const ChartHypothesis* hypoA, const ChartHypothesis* hypoB) const { - // assert in same cell - assert(hypoA->GetCurrSourceRange() == hypoB->GetCurrSourceRange()); - - // shouldn't be mixing hypos with different lhs - assert(hypoA->GetTargetLHS() == hypoB->GetTargetLHS()); - - return (hypoA->RecombineCompare(*hypoB) < 0); - } -}; - /** Contains a set of unique hypos that have the same HS non-term. * ie. 1 of these for each target LHS in each cell */ @@ -64,7 +47,8 @@ class ChartHypothesisCollection friend std::ostream& operator<<(std::ostream&, const ChartHypothesisCollection&); protected: - typedef std::set<ChartHypothesis*, ChartHypothesisRecombinationOrderer> HCType; + //typedef std::set<ChartHypothesis*, ChartHypothesisRecombinationOrderer> HCType; + typedef boost::unordered_set< ChartHypothesis*, UnorderedComparer<ChartHypothesis>, UnorderedComparer<ChartHypothesis> > HCType; HCType m_hypos; HypoList m_hyposOrdered; diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index 296fd9768..504cf605b 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -619,7 +619,7 @@ void ChartManager::OutputDetailedTranslationReport( const StaticData &staticData = StaticData::Instance(); if (staticData.IsDetailedAllTranslationReportingEnabled()) { - const Sentence &sentence = dynamic_cast<const Sentence &>(m_source); + const Sentence &sentence = static_cast<const Sentence &>(m_source); size_t nBestSize = staticData.options().nbest.nbest_size; std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList; CalcNBest(nBestSize, nBestList, staticData.options().nbest.nbest_size); @@ -720,7 +720,7 @@ void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector std::ostringstream out; ApplicationContext applicationContext; - const Sentence &sentence = dynamic_cast<const Sentence &>(m_source); + const Sentence &sentence = static_cast<const Sentence &>(m_source); const size_t translationId = m_source.GetTranslationId(); OutputTreeFragmentsTranslationOptions(out, applicationContext, hypo, sentence, translationId); @@ -731,7 +731,7 @@ void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions(); for( size_t i=0; i<sff.size(); i++ ) { if (sff[i] == treeStructure) { - const TreeState* tree = dynamic_cast<const TreeState*>(hypo->GetFFState(i)); + const TreeState* tree = static_cast<const TreeState*>(hypo->GetFFState(i)); out << "Full Tree " << translationId << ": " << tree->GetTree()->GetString() << "\n"; break; } diff --git a/moses/ChartParser.cpp b/moses/ChartParser.cpp index 98ff9bd00..b129419f6 100644 --- a/moses/ChartParser.cpp +++ b/moses/ChartParser.cpp @@ -44,7 +44,7 @@ ChartParserUnknown ChartParserUnknown::~ChartParserUnknown() { RemoveAllInColl(m_unksrcs); - RemoveAllInColl(m_cacheTargetPhraseCollection); + // RemoveAllInColl(m_cacheTargetPhraseCollection); } void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range, ChartParserCallback &to) @@ -232,11 +232,11 @@ void ChartParser::CreateInputPaths(const InputType &input) InputPath *node; if (range.GetNumWordsCovered() == 1) { - node = new InputPath(subphrase, labels, range, NULL, NULL); + node = new InputPath(m_ttask, subphrase, labels, range, NULL, NULL); vec.push_back(node); } else { const InputPath &prevNode = GetInputPath(startPos, endPos - 1); - node = new InputPath(subphrase, labels, range, &prevNode, NULL); + node = new InputPath(m_ttask, subphrase, labels, range, &prevNode, NULL); vec.push_back(node); } diff --git a/moses/ChartParser.h b/moses/ChartParser.h index 372a05f60..a1d0752c2 100644 --- a/moses/ChartParser.h +++ b/moses/ChartParser.h @@ -27,7 +27,7 @@ #include "WordsRange.h" #include "StackVec.h" #include "InputPath.h" - +#include "TargetPhraseCollection.h" namespace Moses { @@ -38,7 +38,7 @@ class Sentence; class ChartCellCollectionBase; class Word; class Phrase; -class TargetPhraseCollection; +// class TargetPhraseCollection; class DecodeGraph; class ChartParserUnknown @@ -56,7 +56,7 @@ public: private: std::vector<Phrase*> m_unksrcs; - std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection; + std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection; }; class ChartParser diff --git a/moses/ChartParserCallback.h b/moses/ChartParserCallback.h index 9b03e1f5b..7c53654a6 100644 --- a/moses/ChartParserCallback.h +++ b/moses/ChartParserCallback.h @@ -3,6 +3,7 @@ #include "StackVec.h" #include <list> +#include "TargetPhraseCollection.h" namespace Moses { @@ -23,7 +24,7 @@ public: virtual bool Empty() const = 0; - virtual void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range) = 0; + virtual void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const WordsRange &range) = 0; virtual void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath) = 0; diff --git a/moses/ChartTranslationOptionList.cpp b/moses/ChartTranslationOptionList.cpp index 8d3d9b3ab..1d5c261bc 100644 --- a/moses/ChartTranslationOptionList.cpp +++ b/moses/ChartTranslationOptionList.cpp @@ -115,9 +115,13 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc, } } -void ChartTranslationOptionList::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range) +void +ChartTranslationOptionList:: +AddPhraseOOV(TargetPhrase &phrase, + std::list<TargetPhraseCollection::shared_ptr > &waste_memory, + const WordsRange &range) { - TargetPhraseCollection *tpc = new TargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpc(new TargetPhraseCollection); tpc->Add(&phrase); waste_memory.push_back(tpc); StackVec empty; diff --git a/moses/ChartTranslationOptionList.h b/moses/ChartTranslationOptionList.h index 4723bdd1d..c8ca5760c 100644 --- a/moses/ChartTranslationOptionList.h +++ b/moses/ChartTranslationOptionList.h @@ -55,7 +55,7 @@ public: void Add(const TargetPhraseCollection &, const StackVec &, const WordsRange &); - void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range); + void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const WordsRange &range); bool Empty() const { return m_size == 0; diff --git a/moses/ConfusionNet.cpp b/moses/ConfusionNet.cpp index 16db99c44..671f0d166 100644 --- a/moses/ConfusionNet.cpp +++ b/moses/ConfusionNet.cpp @@ -70,7 +70,7 @@ ConfusionNet() : InputType() if (SD.IsSyntax()) { m_defaultLabelSet.insert(SD.GetInputDefaultNonTerminal()); } - UTIL_THROW_IF2(&InputFeature::Instance() == NULL, "Input feature must be specified"); + UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified"); } ConfusionNet:: @@ -140,9 +140,9 @@ ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder) Clear(); // const StaticData &staticData = StaticData::Instance(); - const InputFeature &inputFeature = InputFeature::Instance(); - size_t numInputScores = inputFeature.GetNumInputScores(); - size_t numRealWordCount = inputFeature.GetNumRealWordsInInput(); + const InputFeature *inputFeature = InputFeature::InstancePtr(); + size_t numInputScores = inputFeature->GetNumInputScores(); + size_t numRealWordCount = inputFeature->GetNumRealWordsInInput(); size_t totalCount = numInputScores + numRealWordCount; bool addRealWordCount = (numRealWordCount > 0); diff --git a/moses/ContextScope.h b/moses/ContextScope.h index e1173cbdc..18880db13 100644 --- a/moses/ContextScope.h +++ b/moses/ContextScope.h @@ -12,6 +12,15 @@ #include <boost/foreach.hpp> #endif +// for some reason, the xmlrpc_c headers must be included AFTER the +// boost thread-related ones ... +#ifdef HAVE_XMLRPC_C +#include <xmlrpc-c/base.hpp> +#include <xmlrpc-c/registry.hpp> +#include <xmlrpc-c/server_abyss.hpp> +#endif + + #include <map> #include <boost/shared_ptr.hpp> // #include "thread_safe_container.h" @@ -29,6 +38,7 @@ protected: #ifdef WITH_THREADS mutable boost::shared_mutex m_lock; #endif + SPTR<std::map<std::string,float> > m_context_weights; public: // class write_access // { @@ -99,6 +109,22 @@ public: m_scratchpad = other.m_scratchpad; } +#ifdef HAVE_XMLRPC_C + SPTR<std::map<std::string,float> > + GetContextWeights(xmlrpc_c::value const* spec = NULL) { + if (spec && m_context_weights == NULL) { + boost::unique_lock<boost::shared_mutex> lock(m_lock); + m_context_weights.reset(new std::map<std::string, float>); + + typedef std::map<std::string,xmlrpc_c::value> tmap; + tmap const tmp = static_cast<tmap>(xmlrpc_c::value_struct(*spec)); + for(tmap::const_iterator m = tmp.begin(); m != tmp.end(); ++m) + (*m_context_weights)[m->first] = xmlrpc_c::value_double(m->second); + } + return m_context_weights; + } +#endif + }; }; diff --git a/moses/DecodeStepTranslation.cpp b/moses/DecodeStepTranslation.cpp index 034c06fc2..071dd1cab 100644 --- a/moses/DecodeStepTranslation.cpp +++ b/moses/DecodeStepTranslation.cpp @@ -49,7 +49,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO , PartialTranslOptColl &outputPartialTranslOptColl , TranslationOptionCollection *toc , bool adhereTableLimit - , const TargetPhraseCollection *phraseColl) const + , TargetPhraseCollection::shared_ptr phraseColl) const { if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) { // word deletion @@ -105,7 +105,7 @@ void DecodeStepTranslation::ProcessInitialTranslation( ,PartialTranslOptColl &outputPartialTranslOptColl , size_t startPos, size_t endPos, bool adhereTableLimit , const InputPath &inputPath - , const TargetPhraseCollection *phraseColl) const + , TargetPhraseCollection::shared_ptr phraseColl) const { const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature(); const size_t tableLimit = phraseDictionary->GetTableLimit(); @@ -147,7 +147,8 @@ void DecodeStepTranslation::ProcessInitialTranslationLEGACY( const size_t tableLimit = phraseDictionary->GetTableLimit(); const WordsRange wordsRange(startPos, endPos); - const TargetPhraseCollectionWithSourcePhrase *phraseColl = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,wordsRange); + TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl + = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,wordsRange); if (phraseColl != NULL) { IFVERBOSE(3) { @@ -237,8 +238,8 @@ ProcessLEGACY(TranslationOption const& in, size_t const currSize = inPhrase.GetSize(); size_t const tableLimit = pdict->GetTableLimit(); - TargetPhraseCollectionWithSourcePhrase const* phraseColl; - phraseColl = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange); + TargetPhraseCollectionWithSourcePhrase::shared_ptr phraseColl + = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange); if (phraseColl != NULL) { TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd; diff --git a/moses/DecodeStepTranslation.h b/moses/DecodeStepTranslation.h index 2d381e219..eceebb940 100644 --- a/moses/DecodeStepTranslation.h +++ b/moses/DecodeStepTranslation.h @@ -48,7 +48,7 @@ public: , PartialTranslOptColl &outputPartialTranslOptColl , TranslationOptionCollection *toc , bool adhereTableLimit - , const TargetPhraseCollection *phraseColl) const; + , TargetPhraseCollection::shared_ptr phraseColl) const; /*! initialize list of partial translation options by applying the first translation step @@ -58,7 +58,7 @@ public: , PartialTranslOptColl &outputPartialTranslOptColl , size_t startPos, size_t endPos, bool adhereTableLimit , const InputPath &inputPath - , const TargetPhraseCollection *phraseColl) const; + , TargetPhraseCollection::shared_ptr phraseColl) const; // legacy void ProcessInitialTranslationLEGACY(const InputType &source diff --git a/moses/FF/BleuScoreFeature.cpp b/moses/FF/BleuScoreFeature.cpp index a98964386..5cd83732a 100644 --- a/moses/FF/BleuScoreFeature.cpp +++ b/moses/FF/BleuScoreFeature.cpp @@ -22,32 +22,27 @@ BleuScoreState::BleuScoreState(): m_words(1), { } -int BleuScoreState::Compare(const FFState& o) const +size_t BleuScoreState::hash() const { - if (&o == this) - return 0; - if (StaticData::Instance().IsSyntax()) return 0; - const BleuScoreState& other = dynamic_cast<const BleuScoreState&>(o); - int c = m_words.Compare(other.m_words); - if (c != 0) - return c; - - /*for(size_t i = 0; i < m_ngram_counts.size(); i++) { - if (m_ngram_counts[i] < other.m_ngram_counts[i]) - return -1; - if (m_ngram_counts[i] > other.m_ngram_counts[i]) - return 1; - if (m_ngram_matches[i] < other.m_ngram_matches[i]) - return -1; - if (m_ngram_matches[i] > other.m_ngram_matches[i]) - return 1; - }*/ + size_t ret = hash_value(m_words); + return ret; +} - return 0; +bool BleuScoreState::operator==(const FFState& o) const +{ + if (&o == this) + return true; + + if (StaticData::Instance().IsSyntax()) + return true; + + const BleuScoreState& other = static_cast<const BleuScoreState&>(o); + return m_words == other.m_words; } + std::ostream& operator<<(std::ostream& out, const BleuScoreState& state) { state.print(out); @@ -508,7 +503,7 @@ FFState* BleuScoreFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, if (!m_enabled) return new BleuScoreState(); NGrams::const_iterator reference_ngrams_iter; - const BleuScoreState& ps = dynamic_cast<const BleuScoreState&>(*prev_state); + const BleuScoreState& ps = static_cast<const BleuScoreState&>(*prev_state); BleuScoreState* new_state = new BleuScoreState(ps); float old_bleu, new_bleu; @@ -584,13 +579,13 @@ FFState* BleuScoreFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo, new_state = new BleuScoreState(); else { const FFState* prev_state_zero = cur_hypo.GetPrevHypo(0)->GetFFState(featureID); - const BleuScoreState& ps_zero = dynamic_cast<const BleuScoreState&>(*prev_state_zero); + const BleuScoreState& ps_zero = static_cast<const BleuScoreState&>(*prev_state_zero); new_state = new BleuScoreState(ps_zero); num_words_first_prev = ps_zero.m_target_length; for (size_t i = 0; i < cur_hypo.GetPrevHypos().size(); ++i) { const FFState* prev_state = cur_hypo.GetPrevHypo(i)->GetFFState(featureID); - const BleuScoreState* ps = dynamic_cast<const BleuScoreState*>(prev_state); + const BleuScoreState* ps = static_cast<const BleuScoreState*>(prev_state); BleuScoreState* ps_nonConst = const_cast<BleuScoreState*>(ps); // cerr << "prev phrase: " << cur_hypo.GetPrevHypo(i)->GetOutputPhrase() // << " ( " << cur_hypo.GetPrevHypo(i)->GetTargetLHS() << ")" << endl; diff --git a/moses/FF/BleuScoreFeature.h b/moses/FF/BleuScoreFeature.h index e1a7f09c7..50984a71f 100644 --- a/moses/FF/BleuScoreFeature.h +++ b/moses/FF/BleuScoreFeature.h @@ -25,7 +25,9 @@ public: static size_t bleu_order; BleuScoreState(); - virtual int Compare(const FFState& other) const; + size_t hash() const; + virtual bool operator==(const FFState& other) const; + void print(std::ostream& out) const; private: diff --git a/moses/FF/ConstrainedDecoding.cpp b/moses/FF/ConstrainedDecoding.cpp index 5485c401a..2ae2bffcc 100644 --- a/moses/FF/ConstrainedDecoding.cpp +++ b/moses/FF/ConstrainedDecoding.cpp @@ -22,10 +22,16 @@ ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo) hypo.GetOutputPhrase(m_outputPhrase); } -int ConstrainedDecodingState::Compare(const FFState& other) const +size_t ConstrainedDecodingState::hash() const +{ + size_t ret = hash_value(m_outputPhrase); + return ret; +} + +bool ConstrainedDecodingState::operator==(const FFState& other) const { const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other); - int ret = m_outputPhrase.Compare(otherFF.m_outputPhrase); + bool ret = m_outputPhrase == otherFF.m_outputPhrase; return ret; } diff --git a/moses/FF/ConstrainedDecoding.h b/moses/FF/ConstrainedDecoding.h index 67833a1b4..d3c5713d3 100644 --- a/moses/FF/ConstrainedDecoding.h +++ b/moses/FF/ConstrainedDecoding.h @@ -17,7 +17,8 @@ public: ConstrainedDecodingState(const Hypothesis &hypo); ConstrainedDecodingState(const ChartHypothesis &hypo); - int Compare(const FFState& other) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; const Phrase &GetPhrase() const { return m_outputPhrase; diff --git a/moses/FF/ControlRecombination.cpp b/moses/FF/ControlRecombination.cpp index f7231d9b0..10c2898b1 100644 --- a/moses/FF/ControlRecombination.cpp +++ b/moses/FF/ControlRecombination.cpp @@ -33,18 +33,29 @@ ControlRecombinationState::ControlRecombinationState(const ChartHypothesis &hypo } } -int ControlRecombinationState::Compare(const FFState& other) const +size_t ControlRecombinationState::hash() const +{ + size_t ret; + if (m_ff.GetType() == SameOutput) { + ret = hash_value(m_outputPhrase); + } else { + // compare hypo address. Won't be equal unless they're actually the same hypo + ret = (size_t) m_hypo; + } + return ret; +} + +bool ControlRecombinationState::operator==(const FFState& other) const { const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other); if (m_ff.GetType() == SameOutput) { - int ret = m_outputPhrase.Compare(otherFF.m_outputPhrase); - return ret; + return m_outputPhrase == otherFF.m_outputPhrase; } else { // compare hypo address. Won't be equal unless they're actually the same hypo if (m_hypo == otherFF.m_hypo) - return 0; - return (m_hypo < otherFF.m_hypo) ? -1 : +1; + return true; + return (m_hypo == otherFF.m_hypo); } } diff --git a/moses/FF/ControlRecombination.h b/moses/FF/ControlRecombination.h index f221f772f..106e8a4bc 100644 --- a/moses/FF/ControlRecombination.h +++ b/moses/FF/ControlRecombination.h @@ -26,7 +26,8 @@ public: ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff); ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff); - int Compare(const FFState& other) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; const Phrase &GetPhrase() const { return m_outputPhrase; diff --git a/moses/FF/CoveredReferenceFeature.cpp b/moses/FF/CoveredReferenceFeature.cpp index dd2c890d7..94c6e9c12 100644 --- a/moses/FF/CoveredReferenceFeature.cpp +++ b/moses/FF/CoveredReferenceFeature.cpp @@ -1,6 +1,8 @@ +#include <boost/functional/hash.hpp> #include <vector> #include <algorithm> #include <iterator> +#include <boost/foreach.hpp> #include "CoveredReferenceFeature.h" #include "moses/ScoreComponentCollection.h" #include "moses/Hypothesis.h" @@ -17,29 +19,17 @@ using namespace std; namespace Moses { -int CoveredReferenceState::Compare(const FFState& other) const +size_t CoveredReferenceState::hash() const { - const CoveredReferenceState &otherState = static_cast<const CoveredReferenceState&>(other); - - if (m_coveredRef.size() != otherState.m_coveredRef.size()) { - return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1; - } else { - multiset<string>::const_iterator thisIt, otherIt; - for (thisIt = m_coveredRef.begin(), otherIt = otherState.m_coveredRef.begin(); - thisIt != m_coveredRef.end(); - thisIt++, otherIt++) { - if (*thisIt != *otherIt) return thisIt->compare(*otherIt); - } - } - return 0; - -// return m_coveredRef == otherState.m_coveredRef; + UTIL_THROW2("TODO:Haven't figure this out yet"); +} -// if (m_coveredRef == otherState.m_coveredRef) -// return 0; -// return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1; +bool CoveredReferenceState::operator==(const FFState& other) const +{ + UTIL_THROW2("TODO:Haven't figure this out yet"); } +////////////////////////////////////////////////////////////////////////////////////////////////////////////// void CoveredReferenceFeature::EvaluateInIsolation(const Phrase &source , const TargetPhrase &targetPhrase , ScoreComponentCollection &scoreBreakdown diff --git a/moses/FF/CoveredReferenceFeature.h b/moses/FF/CoveredReferenceFeature.h index d5873f33e..e45f234ae 100644 --- a/moses/FF/CoveredReferenceFeature.h +++ b/moses/FF/CoveredReferenceFeature.h @@ -19,7 +19,9 @@ class CoveredReferenceState : public FFState public: std::multiset<std::string> m_coveredRef; - int Compare(const FFState& other) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; + }; class CoveredReferenceFeature : public StatefulFeatureFunction diff --git a/moses/FF/DeleteRules.cpp b/moses/FF/DeleteRules.cpp new file mode 100644 index 000000000..96bb07c58 --- /dev/null +++ b/moses/FF/DeleteRules.cpp @@ -0,0 +1,90 @@ +#include <vector> +#include "DeleteRules.h" +#include "moses/ScoreComponentCollection.h" +#include "moses/TargetPhrase.h" +#include "moses/InputFileStream.h" +#include "util/exception.hh" + +using namespace std; + +namespace Moses +{ +DeleteRules::DeleteRules(const std::string &line) + :StatelessFeatureFunction(1, line) +{ + m_tuneable = false; + ReadParameters(); +} + +void DeleteRules::Load() +{ + std::vector<FactorType> factorOrder; + factorOrder.push_back(0); // unfactored for now + + InputFileStream strme(m_path); + + string line; + while (getline(strme, line)) { + vector<string> toks = TokenizeMultiCharSeparator(line, "|||"); + UTIL_THROW_IF2(toks.size() != 2, "Line must be source ||| target"); + Phrase source, target; + source.CreateFromString(Input, factorOrder, toks[0], NULL); + target.CreateFromString(Output, factorOrder, toks[1], NULL); + + size_t hash = 0; + boost::hash_combine(hash, source); + boost::hash_combine(hash, target); + m_ruleHashes.insert(hash); + } +} + +void DeleteRules::EvaluateInIsolation(const Phrase &source + , const TargetPhrase &target + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedFutureScore) const +{ + // dense scores + size_t hash = 0; + boost::hash_combine(hash, source); + boost::hash_combine(hash, target); + + boost::unordered_set<size_t>::const_iterator iter; + iter = m_ruleHashes.find(hash); + if (iter != m_ruleHashes.end()) { + scoreBreakdown.PlusEquals(this, -std::numeric_limits<float>::infinity()); + } + +} + +void DeleteRules::EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedFutureScore) const +{} + +void DeleteRules::EvaluateTranslationOptionListWithSourceContext(const InputType &input + + , const TranslationOptionList &translationOptionList) const +{} + +void DeleteRules::EvaluateWhenApplied(const Hypothesis& hypo, + ScoreComponentCollection* accumulator) const +{} + +void DeleteRules::EvaluateWhenApplied(const ChartHypothesis &hypo, + ScoreComponentCollection* accumulator) const +{} + +void DeleteRules::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "path") { + m_path = value; + } else { + StatelessFeatureFunction::SetParameter(key, value); + } +} + +} + diff --git a/moses/FF/DeleteRules.h b/moses/FF/DeleteRules.h new file mode 100644 index 000000000..d02490369 --- /dev/null +++ b/moses/FF/DeleteRules.h @@ -0,0 +1,49 @@ +#pragma once + +#include <string> +#include <boost/unordered_set.hpp> +#include "StatelessFeatureFunction.h" + +namespace Moses +{ + +class DeleteRules : public StatelessFeatureFunction +{ +protected: + std::string m_path; + boost::unordered_set<size_t> m_ruleHashes; +public: + DeleteRules(const std::string &line); + + void Load(); + + bool IsUseable(const FactorMask &mask) const { + return true; + } + + void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedFutureScore) const; + void EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedFutureScore = NULL) const; + + void EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const; + + void EvaluateWhenApplied(const Hypothesis& hypo, + ScoreComponentCollection* accumulator) const; + void EvaluateWhenApplied(const ChartHypothesis &hypo, + ScoreComponentCollection* accumulator) const; + + + void SetParameter(const std::string& key, const std::string& value); + +}; + +} + diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp index e1571d2a9..a565d2b75 100644 --- a/moses/FF/DistortionScoreProducer.cpp +++ b/moses/FF/DistortionScoreProducer.cpp @@ -13,13 +13,16 @@ struct DistortionState_traditional : public FFState { WordsRange range; int first_gap; DistortionState_traditional(const WordsRange& wr, int fg) : range(wr), first_gap(fg) {} - int Compare(const FFState& other) const { + + size_t hash() const { + return range.GetEndPos(); + } + virtual bool operator==(const FFState& other) const { const DistortionState_traditional& o = static_cast<const DistortionState_traditional&>(other); - if (range.GetEndPos() < o.range.GetEndPos()) return -1; - if (range.GetEndPos() > o.range.GetEndPos()) return 1; - return 0; + return range.GetEndPos() == o.range.GetEndPos(); } + }; std::vector<const DistortionScoreProducer*> DistortionScoreProducer::s_staticColl; diff --git a/moses/FF/FFState.h b/moses/FF/FFState.h index 8d9c61b4e..ffecb2e8a 100644 --- a/moses/FF/FFState.h +++ b/moses/FF/FFState.h @@ -2,7 +2,8 @@ #define moses_FFState_h #include <vector> - +#include <stddef.h> +#include "util/exception.hh" namespace Moses { @@ -11,16 +12,27 @@ class FFState { public: virtual ~FFState(); - virtual int Compare(const FFState& other) const = 0; + virtual size_t hash() const = 0; + virtual bool operator==(const FFState& other) const = 0; + + virtual bool operator!=(const FFState& other) const { + return !(*this == other); + } }; class DummyState : public FFState { public: DummyState() {} - int Compare(const FFState& other) const { + + virtual size_t hash() const { return 0; } + + virtual bool operator==(const FFState& other) const { + return true; + } + }; } diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index 9fbf7c83f..d2c3cf639 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -6,6 +6,7 @@ #include "moses/TranslationModel/PhraseDictionaryMemory.h" #include "moses/TranslationModel/PhraseDictionaryMultiModel.h" #include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h" +#include "moses/TranslationModel/PhraseDictionaryGroup.h" #include "moses/TranslationModel/PhraseDictionaryScope3.h" #include "moses/TranslationModel/PhraseDictionaryTransliteration.h" #include "moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h" @@ -55,6 +56,7 @@ #include "NieceTerminal.h" #include "SpanLength.h" #include "SyntaxRHS.h" +#include "DeleteRules.h" #include "moses/FF/SkeletonStatelessFF.h" #include "moses/FF/SkeletonStatefulFF.h" @@ -71,6 +73,7 @@ #include "moses/FF/VW/VWFeatureSourceBigrams.h" #include "moses/FF/VW/VWFeatureSourceIndicator.h" #include "moses/FF/VW/VWFeatureSourcePhraseInternal.h" +#include "moses/FF/VW/VWFeatureSourceSenseWindow.h" #include "moses/FF/VW/VWFeatureSourceWindow.h" #include "moses/FF/VW/VWFeatureTargetBigrams.h" #include "moses/FF/VW/VWFeatureTargetIndicator.h" @@ -213,6 +216,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(PhraseDictionaryScope3); MOSES_FNAME(PhraseDictionaryMultiModel); MOSES_FNAME(PhraseDictionaryMultiModelCounts); + MOSES_FNAME(PhraseDictionaryGroup); MOSES_FNAME(PhraseDictionaryALSuffixArray); // MOSES_FNAME(PhraseDictionaryDynSuffixArray); MOSES_FNAME(PhraseDictionaryTransliteration); @@ -262,6 +266,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(SyntaxRHS); MOSES_FNAME(PhraseOrientationFeature); MOSES_FNAME(UnalignedWordCountFeature); + MOSES_FNAME(DeleteRules); MOSES_FNAME(SkeletonStatelessFF); MOSES_FNAME(SkeletonStatefulFF); @@ -275,6 +280,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(VWFeatureSourceBigrams); MOSES_FNAME(VWFeatureSourceIndicator); MOSES_FNAME(VWFeatureSourcePhraseInternal); + MOSES_FNAME(VWFeatureSourceSenseWindow); MOSES_FNAME(VWFeatureSourceWindow); MOSES_FNAME(VWFeatureTargetBigrams); MOSES_FNAME(VWFeatureTargetPhraseInternal); diff --git a/moses/FF/FeatureFunction.cpp b/moses/FF/FeatureFunction.cpp index 08ad26db8..9b1ea473b 100644 --- a/moses/FF/FeatureFunction.cpp +++ b/moses/FF/FeatureFunction.cpp @@ -112,12 +112,10 @@ void FeatureFunction::ParseLine(const std::string &line) if (m_description == "") { size_t index = description_counts.count(nameStub); - ostringstream dstream; - dstream << nameStub; - dstream << index; + string descr = SPrint(nameStub) + SPrint(index); description_counts.insert(nameStub); - m_description = dstream.str(); + m_description = descr; } } diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index ef3fa4691..571e49a33 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -43,7 +43,7 @@ GlobalLexicalModel::~GlobalLexicalModel() // delete words in the hash data structure DoubleHash::const_iterator iter; for(iter = m_hash.begin(); iter != m_hash.end(); iter++ ) { - map< const Word*, float, WordComparer >::const_iterator iter2; + boost::unordered_map< const Word*, float, UnorderedComparer<Word>, UnorderedComparer<Word> >::const_iterator iter2; for(iter2 = iter->second.begin(); iter2 != iter->second.end(); iter2++ ) { delete iter2->first; // delete input word } @@ -134,7 +134,7 @@ float GlobalLexicalModel::ScorePhrase( const TargetPhrase& targetPhrase ) const sum += inputWordHash->second; } - set< const Word*, WordComparer > alreadyScored; // do not score a word twice + boost::unordered_set< const Word*, UnorderedComparer<Word>, UnorderedComparer<Word> > alreadyScored; // do not score a word twice for(size_t inputIndex = 0; inputIndex < input.GetSize(); inputIndex++ ) { const Word& inputWord = input.GetWord( inputIndex ); if ( alreadyScored.find( &inputWord ) == alreadyScored.end() ) { diff --git a/moses/FF/GlobalLexicalModel.h b/moses/FF/GlobalLexicalModel.h index 3d7f559d0..e9b787220 100644 --- a/moses/FF/GlobalLexicalModel.h +++ b/moses/FF/GlobalLexicalModel.h @@ -33,8 +33,10 @@ class InputType; */ class GlobalLexicalModel : public StatelessFeatureFunction { - typedef std::map< const Word*, std::map< const Word*, float, WordComparer >, WordComparer > DoubleHash; - typedef std::map< const Word*, float, WordComparer > SingleHash; + typedef boost::unordered_map< const Word*, + boost::unordered_map< const Word*, float, UnorderedComparer<Word> , UnorderedComparer<Word> >, + UnorderedComparer<Word>, UnorderedComparer<Word> > DoubleHash; + typedef boost::unordered_map< const Word*, float, UnorderedComparer<Word>, UnorderedComparer<Word> > SingleHash; typedef std::map< const TargetPhrase*, float > LexiconCache; struct ThreadLocalStorage { diff --git a/moses/FF/GlobalLexicalModelUnlimited.cpp b/moses/FF/GlobalLexicalModelUnlimited.cpp index 675af2b6b..15eec019c 100644 --- a/moses/FF/GlobalLexicalModelUnlimited.cpp +++ b/moses/FF/GlobalLexicalModelUnlimited.cpp @@ -5,6 +5,7 @@ #include "moses/Hypothesis.h" #include "moses/TranslationTask.h" #include "util/string_piece_hash.hh" +#include "util/string_stream.hh" using namespace std; @@ -131,7 +132,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo } if (m_biasFeature) { - stringstream feature; + util::StringStream feature; feature << "glm_"; feature << targetString; feature << "~"; @@ -165,7 +166,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo if (m_sourceContext) { if (sourceIndex == 0) { // add <s> trigger feature for source - stringstream feature; + util::StringStream feature; feature << "glm_"; feature << targetString; feature << "~"; @@ -183,7 +184,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end(); if (m_unrestricted || contextExists) { - stringstream feature; + util::StringStream feature; feature << "glm_"; feature << targetString; feature << "~"; @@ -304,7 +305,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo } } } else { - stringstream feature; + util::StringStream feature; feature << "glm_"; feature << targetString; feature << "~"; @@ -323,7 +324,7 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger, StringPiece targetWord) const { - stringstream feature; + util::StringStream feature; feature << "glm_"; feature << targetTrigger; feature << ","; diff --git a/moses/FF/GlobalLexicalModelUnlimited.h b/moses/FF/GlobalLexicalModelUnlimited.h index 3507da352..473a3c0e0 100644 --- a/moses/FF/GlobalLexicalModelUnlimited.h +++ b/moses/FF/GlobalLexicalModelUnlimited.h @@ -16,8 +16,6 @@ #include "moses/FactorTypeSet.h" #include "moses/Sentence.h" -#include "moses/FF/FFState.h" - #ifdef WITH_THREADS #include <boost/thread/tss.hpp> #endif @@ -76,10 +74,6 @@ public: void InitializeForInput(ttasksptr const& ttask); - const FFState* EmptyHypothesisState(const InputType &) const { - return new DummyState(); - } - //TODO: This implements the old interface, but cannot be updated because //it appears to be stateful void EvaluateWhenApplied(const Hypothesis& cur_hypo, diff --git a/moses/FF/InputFeature.h b/moses/FF/InputFeature.h index c7b7237aa..1815cd8f4 100644 --- a/moses/FF/InputFeature.h +++ b/moses/FF/InputFeature.h @@ -17,11 +17,8 @@ protected: bool m_legacy; public: - static const InputFeature& Instance() { - return *s_instance; - } - static InputFeature& InstanceNonConst() { - return *s_instance; + static const InputFeature *InstancePtr() { + return s_instance; } InputFeature(const std::string &line); diff --git a/moses/FF/InternalTree.h b/moses/FF/InternalTree.h index a3db3487e..29db0241e 100644 --- a/moses/FF/InternalTree.h +++ b/moses/FF/InternalTree.h @@ -141,9 +141,13 @@ public: return m_tree; } - int Compare(const FFState& other) const { + virtual size_t hash() const { return 0; - }; + } + virtual bool operator==(const FFState& other) const { + return true; + } + }; -}
\ No newline at end of file +} diff --git a/moses/FF/LexicalReordering/LexicalReordering.cpp b/moses/FF/LexicalReordering/LexicalReordering.cpp index fc8258631..4ff88bd15 100644 --- a/moses/FF/LexicalReordering/LexicalReordering.cpp +++ b/moses/FF/LexicalReordering/LexicalReordering.cpp @@ -107,7 +107,7 @@ EvaluateWhenApplied(const Hypothesis& hypo, { VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) START" << std::endl); Scores score(GetNumScoreComponents(), 0); - const LRState *prev = dynamic_cast<const LRState *>(prev_state); + const LRState *prev = static_cast<const LRState *>(prev_state); LRState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out); out->PlusEquals(this, score); diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.cpp b/moses/FF/LexicalReordering/LexicalReorderingState.cpp index 90de3ad9c..1c30e49bb 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingState.cpp @@ -312,23 +312,30 @@ PhraseBasedReorderingState(const LRModel &config, { } -int -PhraseBasedReorderingState:: -Compare(const FFState& o) const +size_t PhraseBasedReorderingState::hash() const { - if (&o == this) return 0; + size_t ret; + ret = hash_value(m_prevRange); + boost::hash_combine(ret, m_direction); - const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o); - if (m_prevRange == other->m_prevRange) { + return ret; +} + +bool PhraseBasedReorderingState::operator==(const FFState& o) const +{ + if (&o == this) return true; + + const PhraseBasedReorderingState &other = static_cast<const PhraseBasedReorderingState&>(o); + if (m_prevRange == other.m_prevRange) { if (m_direction == LRModel::Forward) { - return ComparePrevScores(other->m_prevOption); + int compareScore = ComparePrevScores(other.m_prevOption); + return compareScore == 0; } else { - return 0; + return true; } - } else if (m_prevRange < other->m_prevRange) { - return -1; + } else { + return false; } - return 1; } LRState* @@ -352,17 +359,22 @@ Expand(const TranslationOption& topt, const InputType& input, /////////////////////////// //BidirectionalReorderingState -int -BidirectionalReorderingState:: -Compare(FFState const& o) const +size_t BidirectionalReorderingState::hash() const +{ + size_t ret = m_backward->hash(); + boost::hash_combine(ret, m_forward->hash()); + return ret; +} + +bool BidirectionalReorderingState::operator==(const FFState& o) const { if (&o == this) return 0; BidirectionalReorderingState const &other = static_cast<BidirectionalReorderingState const&>(o); - int cmp = m_backward->Compare(*other.m_backward); - return (cmp < 0) ? -1 : cmp ? 1 : m_forward->Compare(*other.m_forward); + bool ret = (*m_backward == *other.m_backward) && (*m_forward == *other.m_forward); + return ret; } LRState* @@ -390,14 +402,18 @@ HReorderingBackwardState(const LRModel &config, size_t offset) : LRState(config, LRModel::Backward, offset) { } +size_t HReorderingBackwardState::hash() const +{ + size_t ret = m_reoStack.hash(); + return ret; +} -int -HReorderingBackwardState:: -Compare(const FFState& o) const +bool HReorderingBackwardState::operator==(const FFState& o) const { const HReorderingBackwardState& other = static_cast<const HReorderingBackwardState&>(o); - return m_reoStack.Compare(other.m_reoStack); + bool ret = m_reoStack == other.m_reoStack; + return ret; } LRState* @@ -437,18 +453,24 @@ HReorderingForwardState(const HReorderingForwardState *prev, m_coverage.SetValue(topt.GetSourceWordsRange(), true); } -int -HReorderingForwardState:: -Compare(const FFState& o) const +size_t HReorderingForwardState::hash() const { - if (&o == this) return 0; + size_t ret; + ret = hash_value(m_prevRange); + return ret; +} + +bool HReorderingForwardState::operator==(const FFState& o) const +{ + if (&o == this) return true; HReorderingForwardState const& other = static_cast<HReorderingForwardState const&>(o); - return ((m_prevRange == other.m_prevRange) - ? ComparePrevScores(other.m_prevOption) - : (m_prevRange < other.m_prevRange) ? -1 : 1); + int compareScores = ((m_prevRange == other.m_prevRange) + ? ComparePrevScores(other.m_prevOption) + : (m_prevRange < other.m_prevRange) ? -1 : 1); + return compareScores == 0; } // For compatibility with the phrase-based reordering model, scoring is one diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.h b/moses/FF/LexicalReordering/LexicalReorderingState.h index 96b226a4e..51001e0e3 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.h +++ b/moses/FF/LexicalReordering/LexicalReorderingState.h @@ -144,10 +144,6 @@ public: typedef LRModel::ReorderingType ReorderingType; virtual - int - Compare(const FFState& o) const = 0; - - virtual LRState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const = 0; @@ -222,11 +218,9 @@ public: delete m_forward; } - virtual - int - Compare(const FFState& o) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; - virtual LRState* Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const; @@ -249,9 +243,8 @@ public: PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt); - virtual - int - Compare(const FFState& o) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; virtual LRState* @@ -276,8 +269,9 @@ public: HReorderingBackwardState(const HReorderingBackwardState *prev, const TranslationOption &topt, ReorderingStack reoStack); + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; - virtual int Compare(const FFState& o) const; virtual LRState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const; @@ -303,7 +297,9 @@ public: HReorderingForwardState(const HReorderingForwardState *prev, const TranslationOption &topt); - virtual int Compare(const FFState& o) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; + virtual LRState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const; diff --git a/moses/FF/LexicalReordering/ReorderingStack.cpp b/moses/FF/LexicalReordering/ReorderingStack.cpp index 49a723a36..014caece9 100644 --- a/moses/FF/LexicalReordering/ReorderingStack.cpp +++ b/moses/FF/LexicalReordering/ReorderingStack.cpp @@ -9,15 +9,16 @@ namespace Moses { -int ReorderingStack::Compare(const ReorderingStack& o) const +size_t ReorderingStack::hash() const +{ + std::size_t ret = boost::hash_range(m_stack.begin(), m_stack.end()); + return ret; +} + +bool ReorderingStack::operator==(const ReorderingStack& o) const { const ReorderingStack& other = static_cast<const ReorderingStack&>(o); - if (other.m_stack > m_stack) { - return 1; - } else if (other.m_stack < m_stack) { - return -1; - } - return 0; + return m_stack == other.m_stack; } // Method to push (shift element into the stack and reduce if reqd) diff --git a/moses/FF/LexicalReordering/ReorderingStack.h b/moses/FF/LexicalReordering/ReorderingStack.h index 5a5b80d16..47fe8b055 100644 --- a/moses/FF/LexicalReordering/ReorderingStack.h +++ b/moses/FF/LexicalReordering/ReorderingStack.h @@ -27,7 +27,9 @@ private: public: - int Compare(const ReorderingStack& o) const; + size_t hash() const; + bool operator==(const ReorderingStack& other) const; + int ShiftReduce(WordsRange input_span); private: diff --git a/moses/FF/LexicalReordering/SparseReordering.cpp b/moses/FF/LexicalReordering/SparseReordering.cpp index 6c81ca414..f8416a284 100644 --- a/moses/FF/LexicalReordering/SparseReordering.cpp +++ b/moses/FF/LexicalReordering/SparseReordering.cpp @@ -8,6 +8,7 @@ #include "util/file_piece.hh" #include "util/string_piece.hh" +#include "util/string_stream.hh" #include "util/tokenize_piece.hh" #include "LexicalReordering.h" @@ -26,7 +27,7 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId) { static string kSep = "-"; static string name; - ostringstream buf; + util::StringStream buf; // type side position id word reotype if (type == Phrase) { buf << "phr"; @@ -88,7 +89,7 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic ReadWeightMap(i->second); m_useWeightMap = true; for (int reoType=0; reoType<=LRModel::MAX; ++reoType) { - ostringstream buf; + util::StringStream buf; buf << reoType; m_featureMap2.push_back(m_producer->GetFeatureName(buf.str())); } diff --git a/moses/FF/NieceTerminal.cpp b/moses/FF/NieceTerminal.cpp index 6bd65f37c..a467ce2b1 100644 --- a/moses/FF/NieceTerminal.cpp +++ b/moses/FF/NieceTerminal.cpp @@ -1,5 +1,4 @@ #include <vector> -#include <set> #include "NieceTerminal.h" #include "moses/ScoreComponentCollection.h" #include "moses/TargetPhrase.h" @@ -45,7 +44,7 @@ void NieceTerminal::EvaluateWithSourceContext(const InputType &input const Phrase *ruleSource = targetPhrase.GetRuleSource(); assert(ruleSource); - std::set<Word> terms; + boost::unordered_set<Word> terms; for (size_t i = 0; i < ruleSource->GetSize(); ++i) { const Word &word = ruleSource->GetWord(i); if (!word.IsNonTerminal()) { @@ -81,9 +80,9 @@ void NieceTerminal::EvaluateWhenApplied(const ChartHypothesis &hypo, bool NieceTerminal::ContainTerm(const InputType &input, const WordsRange &ntRange, - const std::set<Word> &terms) const + const boost::unordered_set<Word> &terms) const { - std::set<Word>::const_iterator iter; + boost::unordered_set<Word>::const_iterator iter; for (size_t pos = ntRange.GetStartPos(); pos <= ntRange.GetEndPos(); ++pos) { const Word &word = input.GetWord(pos); diff --git a/moses/FF/NieceTerminal.h b/moses/FF/NieceTerminal.h index 2ee019443..008e34212 100644 --- a/moses/FF/NieceTerminal.h +++ b/moses/FF/NieceTerminal.h @@ -1,6 +1,6 @@ #pragma once -#include <set> +#include <boost/unordered_set.hpp> #include <string> #include "StatelessFeatureFunction.h" @@ -46,7 +46,7 @@ protected: bool m_hardConstraint; bool ContainTerm(const InputType &input, const WordsRange &ntRange, - const std::set<Word> &terms) const; + const boost::unordered_set<Word> &terms) const; }; } diff --git a/moses/FF/OSM-Feature/osmHyp.cpp b/moses/FF/OSM-Feature/osmHyp.cpp index f971bbe8c..246d75e55 100644 --- a/moses/FF/OSM-Feature/osmHyp.cpp +++ b/moses/FF/OSM-Feature/osmHyp.cpp @@ -22,24 +22,32 @@ void osmState::saveState(int jVal, int eVal, map <int , string> & gapVal) E = eVal; } -int osmState::Compare(const FFState& otherBase) const +size_t osmState::hash() const +{ + size_t ret = j; + + boost::hash_combine(ret, E); + boost::hash_combine(ret, gap); + boost::hash_combine(ret, lmState.length); + + return ret; +} + +bool osmState::operator==(const FFState& otherBase) const { const osmState &other = static_cast<const osmState&>(otherBase); if (j != other.j) - return (j < other.j) ? -1 : +1; + return false; if (E != other.E) - return (E < other.E) ? -1 : +1; + return false; if (gap != other.gap) - return (gap < other.gap) ? -1 : +1; + return false; + if (lmState.length != other.lmState.length) + return false; - if (lmState.length < other.lmState.length) return -1; - - if (lmState.length > other.lmState.length) return 1; - - return 0; + return true; } - std::string osmState :: getName() const { @@ -157,11 +165,7 @@ int osmHypothesis :: firstOpenGap(vector <int> & coverageVector) string osmHypothesis :: intToString(int num) { - - std::ostringstream stm; - stm<<num; - - return stm.str(); + return SPrint(num); } diff --git a/moses/FF/OSM-Feature/osmHyp.h b/moses/FF/OSM-Feature/osmHyp.h index 88f171188..9a92e0440 100644 --- a/moses/FF/OSM-Feature/osmHyp.h +++ b/moses/FF/OSM-Feature/osmHyp.h @@ -16,7 +16,9 @@ class osmState : public FFState { public: osmState(const lm::ngram::State & val); - int Compare(const FFState& other) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; + void saveState(int jVal, int eVal, std::map <int , std::string> & gapVal); int getJ()const { return j; diff --git a/moses/FF/PhraseBoundaryFeature.cpp b/moses/FF/PhraseBoundaryFeature.cpp index 3fdcf27f9..ff7c7c3c9 100644 --- a/moses/FF/PhraseBoundaryFeature.cpp +++ b/moses/FF/PhraseBoundaryFeature.cpp @@ -3,20 +3,28 @@ #include "moses/Hypothesis.h" #include "moses/TranslationOption.h" #include "moses/InputPath.h" +#include "util/string_stream.hh" using namespace std; namespace Moses { -int PhraseBoundaryState::Compare(const FFState& other) const +size_t PhraseBoundaryState::hash() const { - const PhraseBoundaryState& rhs = dynamic_cast<const PhraseBoundaryState&>(other); - int tgt = Word::Compare(*m_targetWord,*(rhs.m_targetWord)); - if (tgt) return tgt; - return Word::Compare(*m_sourceWord,*(rhs.m_sourceWord)); + size_t ret = hash_value(*m_targetWord); + boost::hash_combine(ret, hash_value(*m_sourceWord)); + + return ret; +} +bool PhraseBoundaryState::operator==(const FFState& other) const +{ + const PhraseBoundaryState& rhs = static_cast<const PhraseBoundaryState&>(other); + bool ret = *m_targetWord == *rhs.m_targetWord && *m_sourceWord == *rhs.m_sourceWord; + return ret; } +///////////////////////////////////////////////////////////////////////////////////// PhraseBoundaryFeature::PhraseBoundaryFeature(const std::string &line) : StatefulFeatureFunction(0, line) { @@ -46,7 +54,7 @@ void PhraseBoundaryFeature::AddFeatures( ScoreComponentCollection* scores) const { for (size_t i = 0; i < factors.size(); ++i) { - ostringstream name; + util::StringStream name; name << side << ":"; name << factors[i]; name << ":"; @@ -70,7 +78,7 @@ FFState* PhraseBoundaryFeature::EvaluateWhenApplied (const Hypothesis& cur_hypo, const FFState* prev_state, ScoreComponentCollection* scores) const { - const PhraseBoundaryState* pbState = dynamic_cast<const PhraseBoundaryState*>(prev_state); + const PhraseBoundaryState* pbState = static_cast<const PhraseBoundaryState*>(prev_state); const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); if (targetPhrase.GetSize() == 0) { return new PhraseBoundaryState(*pbState); diff --git a/moses/FF/PhraseBoundaryFeature.h b/moses/FF/PhraseBoundaryFeature.h index a5b55e1ef..56189d446 100644 --- a/moses/FF/PhraseBoundaryFeature.h +++ b/moses/FF/PhraseBoundaryFeature.h @@ -23,7 +23,8 @@ public: const Word* GetTargetWord() const { return m_targetWord; } - virtual int Compare(const FFState& other) const; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; private: diff --git a/moses/FF/PhraseLengthFeature.cpp b/moses/FF/PhraseLengthFeature.cpp index 0eb0740b8..9dbb0235e 100644 --- a/moses/FF/PhraseLengthFeature.cpp +++ b/moses/FF/PhraseLengthFeature.cpp @@ -3,6 +3,7 @@ #include "moses/Hypothesis.h" #include "moses/ScoreComponentCollection.h" #include "moses/TranslationOption.h" +#include "util/string_stream.hh" namespace Moses { @@ -25,13 +26,13 @@ void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source size_t sourceLength = source.GetSize(); // create feature names - stringstream nameSource; + util::StringStream nameSource; nameSource << "s" << sourceLength; - stringstream nameTarget; + util::StringStream nameTarget; nameTarget << "t" << targetLength; - stringstream nameBoth; + util::StringStream nameBoth; nameBoth << sourceLength << "," << targetLength; // increase feature counts diff --git a/moses/FF/PhraseLengthFeature.h b/moses/FF/PhraseLengthFeature.h index 9e576946f..9ff36750c 100644 --- a/moses/FF/PhraseLengthFeature.h +++ b/moses/FF/PhraseLengthFeature.h @@ -6,7 +6,6 @@ #include <map> #include "StatelessFeatureFunction.h" -#include "moses/FF/FFState.h" #include "moses/Word.h" #include "moses/FactorCollection.h" diff --git a/moses/FF/PhraseOrientationFeature.cpp b/moses/FF/PhraseOrientationFeature.cpp index 0865dcac5..f20663459 100644 --- a/moses/FF/PhraseOrientationFeature.cpp +++ b/moses/FF/PhraseOrientationFeature.cpp @@ -21,8 +21,17 @@ namespace Moses { +size_t PhraseOrientationFeatureState::hash() const +{ + UTIL_THROW2("TODO:Haven't figure this out yet"); +} +bool PhraseOrientationFeatureState::operator==(const FFState& other) const +{ + UTIL_THROW2("TODO:Haven't figure this out yet"); +} +//////////////////////////////////////////////////////////////////////////////// const std::string PhraseOrientationFeature::MORIENT("M"); const std::string PhraseOrientationFeature::SORIENT("S"); const std::string PhraseOrientationFeature::DORIENT("D"); diff --git a/moses/FF/PhraseOrientationFeature.h b/moses/FF/PhraseOrientationFeature.h index ad5b5a15e..786a632fa 100644 --- a/moses/FF/PhraseOrientationFeature.h +++ b/moses/FF/PhraseOrientationFeature.h @@ -99,46 +99,8 @@ public: return m_rightBoundaryNonTerminalR2LScores[2]; } - - int Compare(const FFState& other) const { - if (!m_distinguishStates) { - return 0; - } - - const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other); - - if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet && - !m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) { - return 0; - } - if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) { - return 1; - } - if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) { - return -1; - } - if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) { - return 1; - } - if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) { - return -1; - } - - if (m_leftBoundaryIsSet) { - int compareLeft = CompareLeftBoundaryRecursive(*this, otherState, m_useSparseNT); - if (compareLeft != 0) { - return compareLeft; - } - } - if (m_rightBoundaryIsSet) { - int compareRight = CompareRightBoundaryRecursive(*this, otherState, m_useSparseNT); - if (compareRight != 0) { - return compareRight; - } - } - - return 0; - }; + virtual size_t hash() const; + virtual bool operator==(const FFState& other) const; protected: diff --git a/moses/FF/PhrasePairFeature.cpp b/moses/FF/PhrasePairFeature.cpp index 1e343877c..d531d9a54 100644 --- a/moses/FF/PhrasePairFeature.cpp +++ b/moses/FF/PhrasePairFeature.cpp @@ -7,6 +7,7 @@ #include "moses/TranslationOption.h" #include "moses/InputPath.h" #include "util/string_piece_hash.hh" +#include "util/string_stream.hh" #include "util/exception.hh" using namespace std; @@ -126,7 +127,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input const bool use_topicid_prob = isnt.GetUseTopicIdAndProb(); // compute pair - ostringstream pair; + util::StringStream pair; + pair << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() ); for (size_t i = 1; i < source.GetSize(); ++i) { const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId); @@ -145,7 +147,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input if(use_topicid) { // use topicid as trigger const long topicid = isnt.GetTopicId(); - stringstream feature; + util::StringStream feature; + feature << m_description << "_"; if (topicid == -1) feature << "unk"; @@ -159,13 +162,13 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input // use topic probabilities const vector<string> &topicid_prob = *(isnt.GetTopicIdAndProb()); if (atol(topicid_prob[0].c_str()) == -1) { - stringstream feature; + util::StringStream feature; feature << m_description << "_unk_"; feature << pair.str(); scoreBreakdown.SparsePlusEquals(feature.str(), 1); } else { for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { - stringstream feature; + util::StringStream feature; feature << m_description << "_"; feature << topicid_prob[i]; feature << "_"; @@ -179,7 +182,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input const long docid = isnt.GetDocumentId(); for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { string sourceTrigger = *p; - ostringstream namestr; + util::StringStream namestr; namestr << m_description << "_"; namestr << sourceTrigger; namestr << "_"; @@ -207,7 +210,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end(); if (m_unrestricted || sourceTriggerExists) { - ostringstream namestr; + util::StringStream namestr; namestr << m_description << "_"; namestr << sourceTrigger; namestr << "~"; @@ -237,7 +240,7 @@ void PhrasePairFeature::EvaluateInIsolation(const Phrase &source , ScoreComponentCollection &estimatedFutureScore) const { if (m_simple) { - ostringstream namestr; + util::StringStream namestr; namestr << m_description << "_"; namestr << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() ); for (size_t i = 1; i < source.GetSize(); ++i) { diff --git a/moses/FF/RulePairUnlexicalizedSource.cpp b/moses/FF/RulePairUnlexicalizedSource.cpp index d65810af8..fe1a4f648 100644 --- a/moses/FF/RulePairUnlexicalizedSource.cpp +++ b/moses/FF/RulePairUnlexicalizedSource.cpp @@ -4,7 +4,7 @@ #include "moses/ScoreComponentCollection.h" #include "moses/FactorCollection.h" #include <sstream> - +#include "util/string_stream.hh" using namespace std; @@ -58,7 +58,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source } } - ostringstream namestr; + util::StringStream namestr; for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) { const Word &wordT = targetPhrase.GetWord(posT); diff --git a/moses/FF/SkeletonStatefulFF.cpp b/moses/FF/SkeletonStatefulFF.cpp index 931556007..1bcf65fb1 100644 --- a/moses/FF/SkeletonStatefulFF.cpp +++ b/moses/FF/SkeletonStatefulFF.cpp @@ -7,14 +7,6 @@ using namespace std; namespace Moses { -int SkeletonState::Compare(const FFState& other) const -{ - const SkeletonState &otherState = static_cast<const SkeletonState&>(other); - - if (m_targetLen == otherState.m_targetLen) - return 0; - return (m_targetLen < otherState.m_targetLen) ? -1 : +1; -} //////////////////////////////////////////////////////////////// SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line) diff --git a/moses/FF/SkeletonStatefulFF.h b/moses/FF/SkeletonStatefulFF.h index cc0bc07a0..20160cf7f 100644 --- a/moses/FF/SkeletonStatefulFF.h +++ b/moses/FF/SkeletonStatefulFF.h @@ -15,7 +15,14 @@ public: :m_targetLen(targetLen) { } - int Compare(const FFState& other) const; + virtual size_t hash() const { + return (size_t) m_targetLen; + } + virtual bool operator==(const FFState& o) const { + const SkeletonState& other = static_cast<const SkeletonState&>(o); + return m_targetLen == other.m_targetLen; + } + }; class SkeletonStatefulFF : public StatefulFeatureFunction diff --git a/moses/FF/SoftSourceSyntacticConstraintsFeature.h b/moses/FF/SoftSourceSyntacticConstraintsFeature.h index 9bd7ffb70..7540dad25 100644 --- a/moses/FF/SoftSourceSyntacticConstraintsFeature.h +++ b/moses/FF/SoftSourceSyntacticConstraintsFeature.h @@ -4,7 +4,6 @@ #include <boost/unordered_map.hpp> #include <boost/unordered_set.hpp> #include "StatelessFeatureFunction.h" -#include "FFState.h" #include "moses/Factor.h" namespace Moses diff --git a/moses/FF/SparseHieroReorderingFeature.cpp b/moses/FF/SparseHieroReorderingFeature.cpp index ee9d4b719..14befa56f 100644 --- a/moses/FF/SparseHieroReorderingFeature.cpp +++ b/moses/FF/SparseHieroReorderingFeature.cpp @@ -6,6 +6,7 @@ #include "moses/Sentence.h" #include "util/exception.hh" +#include "util/string_stream.hh" #include "SparseHieroReorderingFeature.h" @@ -177,7 +178,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied( //Iterate through block pairs const Sentence& sentence = - dynamic_cast<const Sentence&>(cur_hypo.GetManager().GetSource()); + static_cast<const Sentence&>(cur_hypo.GetManager().GetSource()); //const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); for (size_t i = 0; i < sourceBlocks.size()-1; ++i) { Block& leftSourceBlock = sourceBlocks[i]; @@ -202,7 +203,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied( targetLeftRulePos < targetRightRulePos))) { isMonotone = false; } - stringstream buf; + util::StringStream buf; buf << "h_"; //sparse reordering, Huck if (m_type == SourceLeft || m_type == SourceCombined) { buf << GetFactor(sourceLeftBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString(); diff --git a/moses/FF/SparseHieroReorderingFeature.h b/moses/FF/SparseHieroReorderingFeature.h index 945402412..132af58b1 100644 --- a/moses/FF/SparseHieroReorderingFeature.h +++ b/moses/FF/SparseHieroReorderingFeature.h @@ -10,7 +10,6 @@ #include "moses/Sentence.h" #include "StatelessFeatureFunction.h" -#include "FFState.h" namespace Moses { diff --git a/moses/FF/TargetBigramFeature.cpp b/moses/FF/TargetBigramFeature.cpp index 6816410f8..b440c08b4 100644 --- a/moses/FF/TargetBigramFeature.cpp +++ b/moses/FF/TargetBigramFeature.cpp @@ -11,12 +11,19 @@ using namespace std; namespace Moses { -int TargetBigramState::Compare(const FFState& other) const +size_t TargetBigramState::hash() const { - const TargetBigramState& rhs = dynamic_cast<const TargetBigramState&>(other); - return Word::Compare(m_word,rhs.m_word); + std::size_t ret = hash_value(m_word); + return ret; +} + +bool TargetBigramState::operator==(const FFState& other) const +{ + const TargetBigramState& rhs = static_cast<const TargetBigramState&>(other); + return m_word == rhs.m_word; } +//////////////////////////////////////////////////////////////////////////////// TargetBigramFeature::TargetBigramFeature(const std::string &line) :StatefulFeatureFunction(0, line) { @@ -68,7 +75,7 @@ FFState* TargetBigramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, const FFState* prev_state, ScoreComponentCollection* accumulator) const { - const TargetBigramState* tbState = dynamic_cast<const TargetBigramState*>(prev_state); + const TargetBigramState* tbState = static_cast<const TargetBigramState*>(prev_state); assert(tbState); // current hypothesis target phrase diff --git a/moses/FF/TargetBigramFeature.h b/moses/FF/TargetBigramFeature.h index f6e965808..d63d71aca 100644 --- a/moses/FF/TargetBigramFeature.h +++ b/moses/FF/TargetBigramFeature.h @@ -20,7 +20,8 @@ public: const Word& GetWord() const { return m_word; } - virtual int Compare(const FFState& other) const; + size_t hash() const; + virtual bool operator==(const FFState& other) const; private: Word m_word; diff --git a/moses/FF/TargetNgramFeature.cpp b/moses/FF/TargetNgramFeature.cpp index 8414e1bc2..d181a8a37 100644 --- a/moses/FF/TargetNgramFeature.cpp +++ b/moses/FF/TargetNgramFeature.cpp @@ -12,31 +12,38 @@ namespace Moses using namespace std; -int TargetNgramState::Compare(const FFState& other) const +size_t TargetNgramState::hash() const { - const TargetNgramState& rhs = dynamic_cast<const TargetNgramState&>(other); - int result; + std::size_t ret = boost::hash_range(m_words.begin(), m_words.end()); + return ret; +} + +bool TargetNgramState::operator==(const FFState& other) const +{ + const TargetNgramState& rhs = static_cast<const TargetNgramState&>(other); + bool result; if (m_words.size() == rhs.m_words.size()) { for (size_t i = 0; i < m_words.size(); ++i) { - result = Word::Compare(m_words[i],rhs.m_words[i]); - if (result != 0) return result; + result = m_words[i] == rhs.m_words[i]; + if (!result) return false; } - return 0; + return true; } else if (m_words.size() < rhs.m_words.size()) { for (size_t i = 0; i < m_words.size(); ++i) { - result = Word::Compare(m_words[i],rhs.m_words[i]); - if (result != 0) return result; + result = m_words[i] == rhs.m_words[i]; + if (!result) return false; } - return -1; + return true; } else { for (size_t i = 0; i < rhs.m_words.size(); ++i) { - result = Word::Compare(m_words[i],rhs.m_words[i]); - if (result != 0) return result; + result = m_words[i] == rhs.m_words[i]; + if (!result) return false; } - return 1; + return true; } } +//////////////////////////////////////////////////////////////////////////// TargetNgramFeature::TargetNgramFeature(const std::string &line) :StatefulFeatureFunction(0, line) { @@ -108,7 +115,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, // extract all ngrams from current hypothesis vector<Word> prev_words(tnState->GetWords()); - stringstream curr_ngram; + util::StringStream curr_ngram; bool skip = false; // include lower order ngrams? @@ -166,7 +173,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, if (cur_hypo.GetWordsBitmap().IsComplete()) { for (size_t n = m_n; n >= smallest_n; --n) { - stringstream last_ngram; + util::StringStream last_ngram; skip = false; for (size_t i = cur_hypo.GetSize() - n + 1; i < cur_hypo.GetSize() && !skip; ++i) appendNgram(cur_hypo.GetWord(i), skip, last_ngram); @@ -176,7 +183,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, accumulator->PlusEquals(this, last_ngram.str(), 1); } } - return NULL; + return new TargetNgramState(); } // prepare new state @@ -196,7 +203,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo, return new TargetNgramState(new_prev_words); } -void TargetNgramFeature::appendNgram(const Word& word, bool& skip, stringstream &ngram) const +void TargetNgramFeature::appendNgram(const Word& word, bool& skip, util::StringStream &ngram) const { // const string& w = word.GetFactor(m_factorType)->GetString(); const StringPiece w = word.GetString(m_factorType); @@ -249,7 +256,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo suffixTerminals++; // everything else else { - stringstream ngram; + util::StringStream ngram; ngram << m_baseName; if (m_factorType == 0) ngram << factorZero; @@ -360,7 +367,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo suffixTerminals = 0; // remove duplicates - stringstream curr_ngram; + util::StringStream curr_ngram; curr_ngram << m_baseName; curr_ngram << (*contextFactor[m_n-2]).GetString(m_factorType); curr_ngram << ":"; @@ -386,7 +393,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo // remove duplicates size_t size = contextFactor.size(); if (makePrefix && makeSuffix && (size <= m_n)) { - stringstream curr_ngram; + util::StringStream curr_ngram; curr_ngram << m_baseName; for (size_t i = 0; i < size; ++i) { curr_ngram << (*contextFactor[i]).GetString(m_factorType); @@ -404,7 +411,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const { - stringstream ngram; + util::StringStream ngram; size_t size = contextFactor.size(); for (size_t k = 0; k < numberOfStartPos; ++k) { size_t max_end = (size < m_n+k+offset)? size: m_n+k+offset; @@ -429,7 +436,7 @@ void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFacto void TargetNgramFeature::MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfEndPos, size_t offset) const { - stringstream ngram; + util::StringStream ngram; for (size_t k = 0; k < numberOfEndPos; ++k) { size_t end_pos = contextFactor.size()-1-k-offset; for (int start_pos=end_pos-1; (start_pos >= 0) && (end_pos-start_pos < m_n); --start_pos) { diff --git a/moses/FF/TargetNgramFeature.h b/moses/FF/TargetNgramFeature.h index 2e9e71db0..571f6d921 100644 --- a/moses/FF/TargetNgramFeature.h +++ b/moses/FF/TargetNgramFeature.h @@ -12,6 +12,7 @@ #include "moses/LM/SingleFactor.h" #include "moses/ChartHypothesis.h" #include "moses/ChartManager.h" +#include "util/string_stream.hh" namespace Moses { @@ -19,11 +20,15 @@ namespace Moses class TargetNgramState : public FFState { public: - TargetNgramState(std::vector<Word> &words): m_words(words) {} + TargetNgramState() {} + + TargetNgramState(const std::vector<Word> &words): m_words(words) {} const std::vector<Word> GetWords() const { return m_words; } - virtual int Compare(const FFState& other) const; + + size_t hash() const; + virtual bool operator==(const FFState& other) const; private: std::vector<Word> m_words; @@ -153,24 +158,42 @@ public: return m_contextSuffix; } - int Compare(const FFState& o) const { + size_t hash() const { + // not sure if this is correct + size_t ret; + + ret = m_startPos; + boost::hash_combine(ret, m_endPos); + boost::hash_combine(ret, m_inputSize); + + // prefix + if (m_startPos > 0) { // not for "<s> ..." + boost::hash_combine(ret, hash_value(GetPrefix())); + } + + if (m_endPos < m_inputSize - 1) { // not for "... </s>" + boost::hash_combine(ret, hash_value(GetSuffix())); + } + + return ret; + } + virtual bool operator==(const FFState& o) const { const TargetNgramChartState &other = static_cast<const TargetNgramChartState &>( o ); // prefix if (m_startPos > 0) { // not for "<s> ..." - int ret = GetPrefix().Compare(other.GetPrefix()); - if (ret != 0) - return ret; + if (GetPrefix() != other.GetPrefix()) + return false; } if (m_endPos < m_inputSize - 1) { // not for "... </s>" - int ret = GetSuffix().Compare(other.GetSuffix()); - if (ret != 0) - return ret; + if (GetSuffix() != other.GetSuffix()) + return false; } - return 0; + return true; } + }; /** Sets the features of observed ngrams. @@ -222,7 +245,7 @@ private: std::string m_baseName; - void appendNgram(const Word& word, bool& skip, std::stringstream& ngram) const; + void appendNgram(const Word& word, bool& skip, util::StringStream& ngram) const; void MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos = 1, size_t offset = 0) const; void MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, diff --git a/moses/FF/TreeStructureFeature.cpp b/moses/FF/TreeStructureFeature.cpp index 108c99143..75b6d8b57 100644 --- a/moses/FF/TreeStructureFeature.cpp +++ b/moses/FF/TreeStructureFeature.cpp @@ -34,7 +34,7 @@ FFState* TreeStructureFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hy if (word.IsNonTerminal()) { size_t nonTermInd = cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos]; const ChartHypothesis *prevHypo = cur_hypo.GetPrevHypo(nonTermInd); - const TreeState* prev = dynamic_cast<const TreeState*>(prevHypo->GetFFState(featureID)); + const TreeState* prev = static_cast<const TreeState*>(prevHypo->GetFFState(featureID)); const TreePointer prev_tree = prev->GetTree(); previous_trees.push_back(prev_tree); } diff --git a/moses/FF/VW/VWFeatureSourceSenseWindow.h b/moses/FF/VW/VWFeatureSourceSenseWindow.h new file mode 100644 index 000000000..5add76c09 --- /dev/null +++ b/moses/FF/VW/VWFeatureSourceSenseWindow.h @@ -0,0 +1,141 @@ +#pragma once + +#include <string> +#include <algorithm> +#include <boost/foreach.hpp> +#include "ThreadLocalByFeatureStorage.h" +#include "VWFeatureSource.h" +#include "moses/Util.h" + +/* + * Produces features from factors in the following format: + * wordsense1:0.25^wordsense1:0.7^wordsense3:0.05 + * + * This is useful e.g. for including different possible word senses as features weighted + * by their probability. + * + * By default, features are extracted from a small context window around the current + * phrase and from within the phrase. + */ + +namespace Moses +{ + +class VWFeatureSourceSenseWindow : public VWFeatureSource +{ +public: + VWFeatureSourceSenseWindow(const std::string &line) + : VWFeatureSource(line), m_tlsSenses(this), m_tlsForms(this), m_lexicalized(true), m_size(DEFAULT_WINDOW_SIZE) { + ReadParameters(); + + // Call this last + VWFeatureBase::UpdateRegister(); + } + + // precompute feature strings for each input sentence + virtual void InitializeForInput(ttasksptr const& ttask) { + InputType const& input = *(ttask->GetSource().get()); + + std::vector<WordSenses>& senses = *m_tlsSenses.GetStored(); + std::vector<std::string>& forms = *m_tlsForms.GetStored(); + senses.clear(); + forms.clear(); + + senses.resize(input.GetSize()); + forms.resize(input.GetSize()); + + for (size_t i = 0; i < input.GetSize(); i++) { + senses[i] = GetSenses(input, i); + forms[i] = m_lexicalized ? GetWordForm(input, i) + "^" : ""; + } + } + + void operator()(const InputType &input + , const InputPath &inputPath + , const WordsRange &sourceRange + , Discriminative::Classifier &classifier) const { + int begin = sourceRange.GetStartPos(); + int end = sourceRange.GetEndPos() + 1; + int inputLen = input.GetSize(); + + const std::vector<WordSenses>& senses = *m_tlsSenses.GetStored(); + const std::vector<std::string>& forms = *m_tlsForms.GetStored(); + + // before current phrase + for (int i = std::max(0, begin - m_size); i < begin; i++) { + BOOST_FOREACH(const Sense &sense, senses[i]) { + classifier.AddLabelIndependentFeature("snsb^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); + classifier.AddLabelIndependentFeature("snsb^" + forms[i] + sense.m_label, sense.m_prob); + } + } + + // within current phrase + for (int i = begin; i < end; i++) { + BOOST_FOREACH(const Sense &sense, senses[i]) { + classifier.AddLabelIndependentFeature("snsin^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); + classifier.AddLabelIndependentFeature("snsin^" + forms[i] + sense.m_label, sense.m_prob); + } + } + + // after current phrase + for (int i = end; i < std::min(end + m_size, inputLen); i++) { + BOOST_FOREACH(const Sense &sense, senses[i]) { + classifier.AddLabelIndependentFeature("snsa^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); + classifier.AddLabelIndependentFeature("snsa^" + forms[i] + sense.m_label, sense.m_prob); + } + } + } + + virtual void SetParameter(const std::string& key, const std::string& value) { + if (key == "size") { + m_size = Scan<size_t>(value); + } else if (key == "lexicalized") { + m_lexicalized = Scan<bool>(value); + } else { + VWFeatureSource::SetParameter(key, value); + } + } + +private: + static const int DEFAULT_WINDOW_SIZE = 3; + + struct Sense { + std::string m_label; + float m_prob; + }; + + typedef std::vector<Sense> WordSenses; + typedef ThreadLocalByFeatureStorage<std::vector<WordSenses> > TLSSenses; + typedef ThreadLocalByFeatureStorage<std::vector<std::string> > TLSWordForms; + + TLSSenses m_tlsSenses; // for each input sentence, contains extracted senses and probs for each word + TLSWordForms m_tlsForms; // word forms for each input sentence + + + std::vector<Sense> GetSenses(const InputType &input, size_t pos) const { + std::string w = GetWord(input, pos); + std::vector<std::string> senseTokens = Tokenize(w, "^"); + + std::vector<Sense> out(senseTokens.size()); + for (size_t i = 0; i < senseTokens.size(); i++) { + std::vector<std::string> senseColumns = Tokenize(senseTokens[i], ":"); + if (senseColumns.size() != 2) { + UTIL_THROW2("VW :: bad format of sense distribution: " << senseTokens[i]); + } + out[i].m_label = senseColumns[0]; + out[i].m_prob = Scan<float>(senseColumns[1]); + } + + return out; + } + + // assuming that word surface form is always factor 0, output the word form + inline std::string GetWordForm(const InputType &input, size_t pos) const { + return input.GetWord(pos).GetString(0).as_string(); + } + + bool m_lexicalized; + int m_size; +}; + +} diff --git a/moses/FF/WordTranslationFeature.cpp b/moses/FF/WordTranslationFeature.cpp index 1059e34de..88d0ef3a4 100644 --- a/moses/FF/WordTranslationFeature.cpp +++ b/moses/FF/WordTranslationFeature.cpp @@ -179,7 +179,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input if (m_simple) { // construct feature name - stringstream featureName; + util::StringStream featureName; featureName << m_description << "_"; featureName << sourceWord; featureName << "~"; @@ -193,7 +193,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input if(use_topicid) { // use topicid as trigger const long topicid = sentence.GetTopicId(); - stringstream feature; + util::StringStream feature; feature << m_description << "_"; if (topicid == -1) feature << "unk"; @@ -209,7 +209,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input // use topic probabilities const vector<string> &topicid_prob = *(input.GetTopicIdAndProb()); if (atol(topicid_prob[0].c_str()) == -1) { - stringstream feature; + util::StringStream feature; feature << m_description << "_unk_"; feature << sourceWord; feature << "~"; @@ -217,7 +217,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input scoreBreakdown.SparsePlusEquals(feature.str(), 1); } else { for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { - stringstream feature; + util::StringStream feature; feature << m_description << "_"; feature << topicid_prob[i]; feature << "_"; @@ -233,7 +233,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input const long docid = input.GetDocumentId(); for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { string sourceTrigger = *p; - stringstream feature; + util::StringStream feature; feature << m_description << "_"; feature << sourceTrigger; feature << "_"; @@ -248,7 +248,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex; if (!m_domainTrigger && globalSourceIndex == 0) { // add <s> trigger feature for source - stringstream feature; + util::StringStream feature; feature << m_description << "_"; feature << "<s>,"; feature << sourceWord; @@ -278,7 +278,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input if (m_domainTrigger) { if (sourceTriggerExists) { - stringstream feature; + util::StringStream feature; feature << m_description << "_"; feature << sourceTrigger; feature << "_"; @@ -288,7 +288,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input scoreBreakdown.SparsePlusEquals(feature.str(), 1); } } else if (m_unrestricted || sourceTriggerExists) { - stringstream feature; + util::StringStream feature; feature << m_description << "_"; if (contextIndex < globalSourceIndex) { feature << sourceTrigger; diff --git a/moses/FF/WordTranslationFeature.h b/moses/FF/WordTranslationFeature.h index 9ca41da46..fee9cb668 100644 --- a/moses/FF/WordTranslationFeature.h +++ b/moses/FF/WordTranslationFeature.h @@ -5,7 +5,6 @@ #include "moses/FactorCollection.h" #include "moses/Sentence.h" -#include "FFState.h" #include "StatelessFeatureFunction.h" namespace Moses @@ -43,10 +42,6 @@ public: void Load(); - const FFState* EmptyHypothesisState(const InputType &) const { - return new DummyState(); - } - void EvaluateWithSourceContext(const InputType &input , const InputPath &inputPath , const TargetPhrase &targetPhrase diff --git a/moses/FeatureVector.cpp b/moses/FeatureVector.cpp index 89c90cfc2..45a198c84 100644 --- a/moses/FeatureVector.cpp +++ b/moses/FeatureVector.cpp @@ -1,23 +1,21 @@ /* - Moses - factored phrase-based language decoder - Copyright (C) 2010 University of Edinburgh + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - */ +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ #include <algorithm> #include <cmath> @@ -31,6 +29,7 @@ #include "FeatureVector.h" #include "util/string_piece_hash.hh" +#include "util/string_stream.hh" using namespace std; @@ -206,7 +205,7 @@ void FVector::save(const string& filename) const { ofstream out(filename.c_str()); if (!out) { - ostringstream msg; + util::StringStream msg; msg << "Unable to open " << filename; throw runtime_error(msg.str()); } diff --git a/moses/FeatureVector.h b/moses/FeatureVector.h index 839e2a5d4..a34eee4ef 100644 --- a/moses/FeatureVector.h +++ b/moses/FeatureVector.h @@ -1,22 +1,21 @@ /* - Moses - factored phrase-based language decoder - Copyright (C) 2010 University of Edinburgh - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - */ + Moses - statistical machine translation system + Copyright (C) 2005-2015 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ #pragma once #ifndef FEATUREVECTOR_H diff --git a/moses/File.h b/moses/File.h index fbf666ef9..43eacee83 100644 --- a/moses/File.h +++ b/moses/File.h @@ -12,6 +12,7 @@ #include <sstream> #include <vector> #include "util/exception.hh" +#include "util/string_stream.hh" #include "TypeDef.h" #include "Util.h" @@ -147,7 +148,7 @@ inline OFF_T fTell(FILE* f) inline void fSeek(FILE* f,OFF_T o) { if(FSEEKO(f,o,SEEK_SET)<0) { - std::stringstream strme; + util::StringStream strme; strme << "ERROR: could not fseeko position " << o <<"\n"; if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n"; UTIL_THROW2(strme.str()); diff --git a/moses/GenerationDictionary.cpp b/moses/GenerationDictionary.cpp index ddb1428d9..40ff28177 100644 --- a/moses/GenerationDictionary.cpp +++ b/moses/GenerationDictionary.cpp @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "InputFileStream.h" #include "StaticData.h" #include "util/exception.hh" +#include "util/string_stream.hh" using namespace std; @@ -84,9 +85,9 @@ void GenerationDictionary::Load() size_t numFeaturesInFile = token.size() - 2; if (numFeaturesInFile < numFeatureValuesInConfig) { - stringstream strme; + util::StringStream strme; strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig - << " feature values, but found " << numFeaturesInFile << std::endl; + << " feature values, but found " << numFeaturesInFile << "\n"; throw strme.str(); } std::vector<float> scores(numFeatureValuesInConfig, 0.0f); diff --git a/moses/GenerationDictionary.h b/moses/GenerationDictionary.h index 67bbe1e91..82aa82426 100644 --- a/moses/GenerationDictionary.h +++ b/moses/GenerationDictionary.h @@ -23,9 +23,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_GenerationDictionary_h #include <list> -#include <map> #include <stdexcept> #include <vector> +#include <boost/unordered_map.hpp> #include "ScoreComponentCollection.h" #include "Phrase.h" #include "TypeDef.h" @@ -36,7 +36,7 @@ namespace Moses class FactorCollection; -typedef std::map < Word , ScoreComponentCollection > OutputWordCollection; +typedef boost::unordered_map < Word , ScoreComponentCollection > OutputWordCollection; // 1st = output phrase // 2nd = log probability (score) @@ -44,7 +44,7 @@ typedef std::map < Word , ScoreComponentCollection > OutputWordCollection; */ class GenerationDictionary : public DecodeFeature { - typedef std::map<const Word* , OutputWordCollection, WordComparer> Collection; + typedef boost::unordered_map<const Word* , OutputWordCollection, UnorderedComparer<Word>, UnorderedComparer<Word> > Collection; protected: static std::vector<GenerationDictionary*> s_staticColl; diff --git a/moses/Hypothesis.cpp b/moses/Hypothesis.cpp index 5a7833c9f..847c14f81 100644 --- a/moses/Hypothesis.cpp +++ b/moses/Hypothesis.cpp @@ -43,15 +43,10 @@ using namespace std; namespace Moses { - -#ifdef USE_HYPO_POOL -ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000); -#endif - Hypothesis:: -Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt) +Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt, const WordsBitmap &bitmap) : m_prevHypo(NULL) - , m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted) + , m_sourceCompleted(bitmap) , m_sourceInput(source) , m_currSourceWordsRange( m_sourceCompleted.GetFirstGapPos()>0 ? 0 : NOT_FOUND, @@ -80,9 +75,9 @@ Hypothesis(Manager& manager, InputType const& source, const TranslationOption &i * continue prevHypo by appending the phrases in transOpt */ Hypothesis:: -Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt) +Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt, const WordsBitmap &bitmap) : m_prevHypo(&prevHypo) - , m_sourceCompleted(prevHypo.m_sourceCompleted ) + , m_sourceCompleted(bitmap) , m_sourceInput(prevHypo.m_sourceInput) , m_currSourceWordsRange(transOpt.GetSourceWordsRange()) , m_currTargetWordsRange(prevHypo.m_currTargetWordsRange.GetEndPos() + 1, @@ -98,13 +93,6 @@ Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt) , m_id(m_manager.GetNextHypoId()) { m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown()); - - // assert that we are not extending our hypothesis by retranslating something - // that this hypothesis has already translated! - assert(!m_sourceCompleted.Overlap(m_currSourceWordsRange)); - - //_hash_computed = false; - m_sourceCompleted.SetValue(m_currSourceWordsRange.GetStartPos(), m_currSourceWordsRange.GetEndPos(), true); m_wordDeleted = transOpt.IsDeletionOption(); m_manager.GetSentenceStats().AddCreated(); } @@ -118,7 +106,7 @@ Hypothesis:: if (m_arcList) { ArcList::iterator iter; for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) { - FREEHYPO(*iter); + delete *iter; } m_arcList->clear(); @@ -153,77 +141,6 @@ AddArc(Hypothesis *loserHypo) m_arcList->push_back(loserHypo); } -/*** - * return the subclass of Hypothesis most appropriate to the given translation option - */ -Hypothesis* -Hypothesis:: -CreateNext(const TranslationOption &transOpt) const -{ - return Create(*this, transOpt); -} - -/*** - * return the subclass of Hypothesis most appropriate to the given translation option - */ -Hypothesis* -Hypothesis:: -Create(const Hypothesis &prevHypo, const TranslationOption &transOpt) -{ - -#ifdef USE_HYPO_POOL - Hypothesis *ptr = s_objectPool.getPtr(); - return new(ptr) Hypothesis(prevHypo, transOpt); -#else - return new Hypothesis(prevHypo, transOpt); -#endif -} -/*** - * return the subclass of Hypothesis most appropriate to the given target phrase - */ - -Hypothesis* -Hypothesis:: -Create(Manager& manager, InputType const& m_source, - const TranslationOption &initialTransOpt) -{ -#ifdef USE_HYPO_POOL - Hypothesis *ptr = s_objectPool.getPtr(); - return new(ptr) Hypothesis(manager, m_source, initialTransOpt); -#else - return new Hypothesis(manager, m_source, initialTransOpt); -#endif -} - -/** check, if two hypothesis can be recombined. - this is actually a sorting function that allows us to - keep an ordered list of hypotheses. This makes recombination - much quicker. -*/ -int -Hypothesis:: -RecombineCompare(const Hypothesis &compare) const -{ - // -1 = this < compare - // +1 = this > compare - // 0 = this ==compare - int comp = m_sourceCompleted.Compare(compare.m_sourceCompleted); - if (comp != 0) - return comp; - - for (unsigned i = 0; i < m_ffStates.size(); ++i) { - if (m_ffStates[i] == NULL || compare.m_ffStates[i] == NULL) { - // TODO: Can this situation actually occur? - comp = int(m_ffStates[i] != NULL) - int(compare.m_ffStates[i] != NULL); - } else { - comp = m_ffStates[i]->Compare(*compare.m_ffStates[i]); - } - if (comp != 0) return comp; - } - - return 0; -} - void Hypothesis:: EvaluateWhenApplied(StatefulFeatureFunction const& sfff, @@ -255,7 +172,7 @@ EvaluateWhenApplied(const StatelessFeatureFunction& slff) */ void Hypothesis:: -EvaluateWhenApplied(const SquareMatrix &futureScore) +EvaluateWhenApplied(float futureScore) { IFVERBOSE(2) { m_manager.GetSentenceStats().StartTimeOtherScore(); @@ -292,7 +209,7 @@ EvaluateWhenApplied(const SquareMatrix &futureScore) } // FUTURE COST - m_futureScore = futureScore.CalcFutureScore( m_sourceCompleted ); + m_futureScore = futureScore; // TOTAL m_totalScore = m_currScoreBreakdown.GetWeightedScore() + m_futureScore; @@ -379,7 +296,7 @@ CleanupArcList() // delete bad ones ArcList::iterator iter; for (iter = m_arcList->begin() + nBestSize; iter != m_arcList->end() ; ++iter) - FREEHYPO(*iter); + delete *iter; m_arcList->erase(m_arcList->begin() + nBestSize, m_arcList->end()); } @@ -647,6 +564,40 @@ GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const return ret; } +size_t Hypothesis::hash() const +{ + size_t seed; + + // coverage + seed = m_sourceCompleted.hash(); + + // states + for (size_t i = 0; i < m_ffStates.size(); ++i) { + const FFState *state = m_ffStates[i]; + size_t hash = state->hash(); + boost::hash_combine(seed, hash); + } + return seed; +} + +bool Hypothesis::operator==(const Hypothesis& other) const +{ + // coverage + if (m_sourceCompleted != other.m_sourceCompleted) { + return false; + } + + // states + for (size_t i = 0; i < m_ffStates.size(); ++i) { + const FFState &thisState = *m_ffStates[i]; + const FFState &otherState = *other.m_ffStates[i]; + if (thisState != otherState) { + return false; + } + } + return true; +} + #ifdef HAVE_XMLRPC_C void Hypothesis:: diff --git a/moses/Hypothesis.h b/moses/Hypothesis.h index 0ce75b83c..df639d251 100644 --- a/moses/Hypothesis.h +++ b/moses/Hypothesis.h @@ -71,13 +71,8 @@ class Hypothesis friend std::ostream& operator<<(std::ostream&, const Hypothesis&); protected: - static ObjectPool<Hypothesis> s_objectPool; - const Hypothesis* m_prevHypo; /*! backpointer to previous hypothesis (from which this one was created) */ -// const Phrase &m_targetPhrase; /*! target phrase being created at the current decoding step */ - WordsBitmap m_sourceCompleted; /*! keeps track of which words have been translated so far */ - //TODO: how to integrate this into confusion network framework; what if - //it's a confusion network in the end??? + const WordsBitmap &m_sourceCompleted; /*! keeps track of which words have been translated so far */ InputType const& m_sourceInput; WordsRange m_currSourceWordsRange; /*! source word positions of the last phrase that was used to create this hypothesis */ WordsRange m_currTargetWordsRange; /*! target word positions of the last phrase that was used to create this hypothesis */ @@ -95,29 +90,13 @@ protected: int m_id; /*! numeric ID of this hypothesis, used for logging */ +public: /*! used by initial seeding of the translation process */ - Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt); + Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt, const WordsBitmap &bitmap); /*! used when creating a new hypothesis using a translation option (phrase translation) */ - Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt); - -public: - static ObjectPool<Hypothesis> &GetObjectPool() { - return s_objectPool; - } - + Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt, const WordsBitmap &bitmap); ~Hypothesis(); - /** return the subclass of Hypothesis most appropriate to the given translation option */ - static Hypothesis* Create(const Hypothesis &prevHypo, const TranslationOption &transOpt); - - static Hypothesis* Create(Manager& manager, const WordsBitmap &initialCoverage); - - /** return the subclass of Hypothesis most appropriate to the given target phrase */ - static Hypothesis* Create(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt); - - /** return the subclass of Hypothesis most appropriate to the given translation option */ - Hypothesis* CreateNext(const TranslationOption &transOpt) const; - void PrintHypothesis() const; const InputType& GetInput() const { @@ -146,7 +125,7 @@ public: return m_currTargetWordsRange.GetNumWordsCovered(); } - void EvaluateWhenApplied(const SquareMatrix &futureScore); + void EvaluateWhenApplied(float futureScore); int GetId()const { return m_id; @@ -197,8 +176,6 @@ public: return m_sourceCompleted.IsComplete(); } - int RecombineCompare(const Hypothesis &compare) const; - void GetOutputPhrase(Phrase &out) const; void ToStream(std::ostream& out) const { @@ -211,7 +188,7 @@ public: if (m_prevHypo != NULL) { m_prevHypo->ToStream(out); } - out << (Phrase) GetCurrTargetPhrase(); + out << (const Phrase&) GetCurrTargetPhrase(); } std::string GetOutputString() const { @@ -288,13 +265,16 @@ public: // creates a map of TARGET positions which should be replaced by word using placeholder std::map<size_t, const Moses::Factor*> GetPlaceholders(const Moses::Hypothesis &hypo, Moses::FactorType placeholderFactor) const; + // for unordered_set in stack + size_t hash() const; + bool operator==(const Hypothesis& other) const; + #ifdef HAVE_XMLRPC_C void OutputWordAlignment(std::vector<xmlrpc_c::value>& out) const; void OutputLocalWordAlignment(std::vector<xmlrpc_c::value>& dest) const; #endif - }; std::ostream& operator<<(std::ostream& out, const Hypothesis& hypothesis); @@ -306,34 +286,5 @@ struct CompareHypothesisTotalScore { } }; -#ifdef USE_HYPO_POOL - -#define FREEHYPO(hypo) \ -{ \ - ObjectPool<Hypothesis> &pool = Hypothesis::GetObjectPool(); \ - pool.freeObject(hypo); \ -} \ - -#else -#define FREEHYPO(hypo) delete hypo -#endif - -/** defines less-than relation on hypotheses. -* The particular order is not important for us, we need just to figure out -* which hypothesis are equal based on: -* the last n-1 target words are the same -* and the covers (source words translated) are the same -* Directly using RecombineCompare is unreliable because the Compare methods -* of some states are based on archictecture-dependent pointer comparisons. -* That's why we use the hypothesis IDs instead. -*/ -class HypothesisRecombinationOrderer -{ -public: - bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const { - return (hypoA->RecombineCompare(*hypoB) < 0); - } -}; - } #endif diff --git a/moses/HypothesisStack.cpp b/moses/HypothesisStack.cpp index dfa03108e..418f129bc 100644 --- a/moses/HypothesisStack.cpp +++ b/moses/HypothesisStack.cpp @@ -22,7 +22,7 @@ void HypothesisStack::Remove(const HypothesisStack::iterator &iter) { Hypothesis *h = *iter; Detach(iter); - FREEHYPO(h); + delete h; } diff --git a/moses/HypothesisStack.h b/moses/HypothesisStack.h index 0c3d4198f..1994c1d7f 100644 --- a/moses/HypothesisStack.h +++ b/moses/HypothesisStack.h @@ -3,6 +3,7 @@ #include <vector> #include <set> +#include <boost/unordered_set.hpp> #include "Hypothesis.h" #include "WordsBitmap.h" @@ -18,7 +19,7 @@ class HypothesisStack { protected: - typedef std::set< Hypothesis*, HypothesisRecombinationOrderer > _HCType; + typedef boost::unordered_set< Hypothesis*, UnorderedComparer<Hypothesis>, UnorderedComparer<Hypothesis> > _HCType; _HCType m_hypos; /**< contains hypotheses */ Manager& m_manager; diff --git a/moses/HypothesisStackCubePruning.cpp b/moses/HypothesisStackCubePruning.cpp index 23fc2b01a..7a18c5ba5 100644 --- a/moses/HypothesisStackCubePruning.cpp +++ b/moses/HypothesisStackCubePruning.cpp @@ -39,6 +39,7 @@ HypothesisStackCubePruning::HypothesisStackCubePruning(Manager& manager) : m_nBestIsEnabled = StaticData::Instance().options().nbest.enabled; m_bestScore = -std::numeric_limits<float>::infinity(); m_worstScore = -std::numeric_limits<float>::infinity(); + m_deterministic = manager.options().cube.deterministic_search; } /** remove all hypotheses from the collection */ @@ -85,7 +86,7 @@ bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo) if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) { m_manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, constraint" << std::endl); - FREEHYPO(hypo); + delete hypo; return false; } @@ -93,7 +94,7 @@ bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo) // too bad for stack. don't bother adding hypo into collection m_manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, too bad for stack" << std::endl); - FREEHYPO(hypo); + delete hypo; return false; } @@ -135,7 +136,7 @@ bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo) if (m_nBestIsEnabled) { hypoExisting->AddArc(hypo); } else { - FREEHYPO(hypo); + delete hypo; } return false; } @@ -148,7 +149,7 @@ void HypothesisStackCubePruning::AddInitial(Hypothesis *hypo) "Should have added hypothesis " << *hypo); const WordsBitmap &bitmap = hypo->GetWordsBitmap(); - m_bitmapAccessor[bitmap] = new BitmapContainer(bitmap, *this); + AddBitmapContainer(bitmap, *this); } void HypothesisStackCubePruning::PruneToSize(size_t newSize) @@ -254,16 +255,7 @@ void HypothesisStackCubePruning::SetBitmapAccessor(const WordsBitmap &newBitmap , const SquareMatrix &futureScore , const TranslationOptionList &transOptList) { - _BMType::iterator bcExists = m_bitmapAccessor.find(newBitmap); - - BitmapContainer *bmContainer; - if (bcExists == m_bitmapAccessor.end()) { - bmContainer = new BitmapContainer(newBitmap, stack); - m_bitmapAccessor[newBitmap] = bmContainer; - } else { - bmContainer = bcExists->second; - } - + BitmapContainer *bmContainer = AddBitmapContainer(newBitmap, stack); BackwardsEdge *edge = new BackwardsEdge(bitmapContainer , *bmContainer , transOptList @@ -301,5 +293,20 @@ HypothesisStackCubePruning::AddHypothesesToBitmapContainers() } } +BitmapContainer *HypothesisStackCubePruning::AddBitmapContainer(const WordsBitmap &bitmap, HypothesisStackCubePruning &stack) +{ + _BMType::iterator iter = m_bitmapAccessor.find(bitmap); + + BitmapContainer *bmContainer; + if (iter == m_bitmapAccessor.end()) { + bmContainer = new BitmapContainer(bitmap, stack, m_deterministic); + m_bitmapAccessor[bitmap] = bmContainer; + } else { + bmContainer = iter->second; + } + + return bmContainer; +} + } diff --git a/moses/HypothesisStackCubePruning.h b/moses/HypothesisStackCubePruning.h index 6dc973ed3..2e3c81cde 100644 --- a/moses/HypothesisStackCubePruning.h +++ b/moses/HypothesisStackCubePruning.h @@ -23,11 +23,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_HypothesisStackCubePruning_h #include <limits> -#include <map> #include <set> +#include <boost/unordered_map.hpp> #include "Hypothesis.h" #include "BitmapContainer.h" #include "HypothesisStack.h" +#include "Util.h" namespace Moses { @@ -36,7 +37,7 @@ class BitmapContainer; class TranslationOptionList; class Manager; -typedef std::map<WordsBitmap, BitmapContainer*> _BMType; +typedef boost::unordered_map<WordsBitmap, BitmapContainer*, UnorderedComparer<WordsBitmap>, UnorderedComparer<WordsBitmap> > _BMType; /** A stack for phrase-based decoding with cube-pruning. */ class HypothesisStackCubePruning : public HypothesisStack @@ -52,6 +53,7 @@ protected: float m_beamWidth; /**< minimum score due to threashold pruning */ size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */ bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */ + bool m_deterministic; /**< flag to determine whether to sort hypotheses deterministically */ /** add hypothesis to stack. Prune if necessary. * Returns false if equiv hypo exists in collection, otherwise returns true @@ -61,6 +63,8 @@ protected: /** destroy all instances of Hypothesis in this collection */ void RemoveAll(); + BitmapContainer *AddBitmapContainer(const WordsBitmap &bitmap, HypothesisStackCubePruning &stack); + public: HypothesisStackCubePruning(Manager& manager); ~HypothesisStackCubePruning() { diff --git a/moses/HypothesisStackNormal.cpp b/moses/HypothesisStackNormal.cpp index e72c803b7..9f13213d4 100644 --- a/moses/HypothesisStackNormal.cpp +++ b/moses/HypothesisStackNormal.cpp @@ -92,7 +92,7 @@ bool HypothesisStackNormal::AddPrune(Hypothesis *hypo) if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) { m_manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, constraint" << std::endl); - FREEHYPO(hypo); + delete hypo; return false; } @@ -103,7 +103,7 @@ bool HypothesisStackNormal::AddPrune(Hypothesis *hypo) && hypo->GetTotalScore() >= GetWorstScoreForBitmap( hypo->GetWordsBitmap() ) ) ) { m_manager.GetSentenceStats().AddDiscarded(); VERBOSE(3,"discarded, too bad for stack" << std::endl); - FREEHYPO(hypo); + delete hypo; return false; } @@ -145,7 +145,7 @@ bool HypothesisStackNormal::AddPrune(Hypothesis *hypo) if (m_nBestIsEnabled) { hypoExisting->AddArc(hypo); } else { - FREEHYPO(hypo); + delete hypo; } return false; } @@ -205,7 +205,7 @@ void HypothesisStackNormal::PruneToSize(size_t newSize) // delete hypotheses that have not been included for(size_t i=0; i<hypos.size(); i++) { if (! included[i]) { - FREEHYPO( hypos[i] ); + delete hypos[i]; m_manager.GetSentenceStats().AddPruning(); } } diff --git a/moses/Incremental.cpp b/moses/Incremental.cpp index d1eb3b532..e0a994ca0 100644 --- a/moses/Incremental.cpp +++ b/moses/Incremental.cpp @@ -83,7 +83,7 @@ public: void Add(const TargetPhraseCollection &targets, const StackVec &nts, const WordsRange &ignored); - void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range); + void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &waste_memory, const WordsRange &range); float GetBestScore(const ChartCellLabel *chartCell) const; @@ -160,7 +160,7 @@ template <class Model> void Fill<Model>::Add(const TargetPhraseCollection &targe } } -template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &range) +template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection::shared_ptr > &, const WordsRange &range) { std::vector<lm::WordIndex> words; UTIL_THROW_IF2(phrase.GetSize() > 1, @@ -450,7 +450,7 @@ void Manager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *coll } const search::Applied *applied = &Completed()[0]; - const Sentence &sentence = dynamic_cast<const Sentence &>(m_source); + const Sentence &sentence = static_cast<const Sentence &>(m_source); const size_t translationId = m_source.GetTranslationId(); std::ostringstream out; diff --git a/moses/InputPath.cpp b/moses/InputPath.cpp index ab7c9c782..b57d70704 100644 --- a/moses/InputPath.cpp +++ b/moses/InputPath.cpp @@ -11,16 +11,19 @@ using namespace std; namespace Moses { InputPath:: -InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, - const WordsRange &range, const InputPath *prevNode, +InputPath(ttaskwptr const theTask, + Phrase const& phrase, + NonTerminalSet const& sourceNonTerms, + WordsRange const& range, InputPath const *prevNode, const ScorePair *inputScore) - :m_prevPath(prevNode) - ,m_phrase(phrase) - ,m_range(range) - ,m_inputScore(inputScore) - ,m_nextNode(1) - ,m_sourceNonTerms(sourceNonTerms) - ,m_sourceNonTermArray(FactorCollection::Instance().GetNumNonTerminals(), false) + : ttask(theTask) + , m_prevPath(prevNode) + , m_phrase(phrase) + , m_range(range) + , m_inputScore(inputScore) + , m_nextNode(1) + , m_sourceNonTerms(sourceNonTerms) + , m_sourceNonTermArray(FactorCollection::Instance().GetNumNonTerminals(), false) { for (NonTerminalSet::const_iterator iter = sourceNonTerms.begin(); iter != sourceNonTerms.end(); ++iter) { size_t idx = (*iter)[0]->GetId(); @@ -33,30 +36,43 @@ InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, InputPath::~InputPath() { - // Since there is no way for the Phrase Dictionaries to tell in - // which (sentence) context phrases were looked up, we tell them - // now that the phrase isn't needed any more by this inputPath - typedef std::pair<const TargetPhraseCollection*, const void* > entry; - std::map<const PhraseDictionary*, entry>::const_iterator iter; - for (iter = m_targetPhrases.begin(); iter != m_targetPhrases.end(); ++iter) - iter->first->Release(iter->second.first); + + // std::cerr << "Deconstructing InputPath" << std::endl; + + + // // NOT NEEDED ANY MORE SINCE THE SWITCH TO SHARED POINTERS + // // Since there is no way for the Phrase Dictionaries to tell in + // // which (sentence) context phrases were looked up, we tell them + // // now that the phrase isn't needed any more by this inputPath + // typedef std::pair<boost::shared_ptr<TargetPhraseCollection>, const void* > entry; + // std::map<const PhraseDictionary*, entry>::iterator iter; + // ttasksptr theTask = this->ttask.lock(); + // for (iter = m_targetPhrases.begin(); iter != m_targetPhrases.end(); ++iter) + // { + // // std::cerr << iter->second.first << " decommissioned." << std::endl; + // iter->first->Release(theTask, iter->second.first); + // } delete m_inputScore; } -const TargetPhraseCollection *InputPath::GetTargetPhrases(const PhraseDictionary &phraseDictionary) const +TargetPhraseCollection::shared_ptr +InputPath:: +GetTargetPhrases(const PhraseDictionary &phraseDictionary) const { - std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> >::const_iterator iter; + TargetPhrases::const_iterator iter; iter = m_targetPhrases.find(&phraseDictionary); if (iter == m_targetPhrases.end()) { - return NULL; + return TargetPhraseCollection::shared_ptr(); } return iter->second.first; } -const void *InputPath::GetPtNode(const PhraseDictionary &phraseDictionary) const +const void* +InputPath:: +GetPtNode(const PhraseDictionary &phraseDictionary) const { - std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> >::const_iterator iter; + TargetPhrases::const_iterator iter; iter = m_targetPhrases.find(&phraseDictionary); if (iter == m_targetPhrases.end()) { return NULL; @@ -64,11 +80,14 @@ const void *InputPath::GetPtNode(const PhraseDictionary &phraseDictionary) const return iter->second.second; } -void InputPath::SetTargetPhrases(const PhraseDictionary &phraseDictionary - , const TargetPhraseCollection *targetPhrases - , const void *ptNode) +void +InputPath:: +SetTargetPhrases(const PhraseDictionary &phraseDictionary, + TargetPhraseCollection::shared_ptr const& targetPhrases, + const void *ptNode) { - std::pair<const TargetPhraseCollection*, const void*> value(targetPhrases, ptNode); + std::pair<TargetPhraseCollection::shared_ptr, const void*> + value(targetPhrases, ptNode); m_targetPhrases[&phraseDictionary] = value; } @@ -83,10 +102,10 @@ const Word &InputPath::GetLastWord() const size_t InputPath::GetTotalRuleSize() const { size_t ret = 0; - std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> >::const_iterator iter; + TargetPhrases::const_iterator iter; for (iter = m_targetPhrases.begin(); iter != m_targetPhrases.end(); ++iter) { // const PhraseDictionary *pt = iter->first; - const TargetPhraseCollection *tpColl = iter->second.first; + TargetPhraseCollection::shared_ptr tpColl = iter->second.first; if (tpColl) { ret += tpColl->GetSize(); @@ -100,10 +119,10 @@ std::ostream& operator<<(std::ostream& out, const InputPath& obj) { out << &obj << " " << obj.GetWordsRange() << " " << obj.GetPrevPath() << " " << obj.GetPhrase(); - std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> >::const_iterator iter; + InputPath::TargetPhrases::const_iterator iter; for (iter = obj.m_targetPhrases.begin(); iter != obj.m_targetPhrases.end(); ++iter) { const PhraseDictionary *pt = iter->first; - const TargetPhraseCollection *tpColl = iter->second.first; + boost::shared_ptr<TargetPhraseCollection const> tpColl = iter->second.first; out << pt << "="; if (tpColl) { diff --git a/moses/InputPath.h b/moses/InputPath.h index c67d88795..84dd22a26 100644 --- a/moses/InputPath.h +++ b/moses/InputPath.h @@ -1,3 +1,4 @@ +// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*- #pragma once #include <map> @@ -7,12 +8,12 @@ #include "WordsRange.h" #include "NonTerminal.h" #include "moses/FactorCollection.h" - +#include <boost/shared_ptr.hpp> +#include "TargetPhraseCollection.h" namespace Moses { class PhraseDictionary; -class TargetPhraseCollection; class ScoreComponentCollection; class TargetPhrase; class InputPath; @@ -31,8 +32,15 @@ class InputPath friend std::ostream& operator<<(std::ostream& out, const InputPath &obj); public: - typedef std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> > TargetPhrases; + typedef std::pair<TargetPhraseCollection::shared_ptr, const void*> + TPCollStoreEntry; + + typedef std::map<const PhraseDictionary*, TPCollStoreEntry> + TargetPhrases; + +public: + ttaskwptr const ttask; protected: const InputPath *m_prevPath; Phrase m_phrase; @@ -57,8 +65,13 @@ public: , m_nextNode(NOT_FOUND) { } - InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, const WordsRange &range, const InputPath *prevNode - ,const ScorePair *inputScore); + InputPath(ttaskwptr const ttask, + Phrase const& phrase, + NonTerminalSet const& sourceNonTerms, + WordsRange const& range, + InputPath const* prevNode, + ScorePair const* inputScore); + ~InputPath(); const Phrase &GetPhrase() const { @@ -88,10 +101,14 @@ public: m_nextNode = nextNode; } - void SetTargetPhrases(const PhraseDictionary &phraseDictionary - , const TargetPhraseCollection *targetPhrases - , const void *ptNode); - const TargetPhraseCollection *GetTargetPhrases(const PhraseDictionary &phraseDictionary) const; + void + SetTargetPhrases(const PhraseDictionary &phraseDictionary, + TargetPhraseCollection::shared_ptr const& targetPhrases, + const void *ptNode); + + TargetPhraseCollection::shared_ptr + GetTargetPhrases(const PhraseDictionary &phraseDictionary) const; + const TargetPhrases &GetTargetPhrases() const { return m_targetPhrases; } diff --git a/moses/LM/BackwardLMState.cpp b/moses/LM/BackwardLMState.cpp index 466c4b655..7324b9460 100644 --- a/moses/LM/BackwardLMState.cpp +++ b/moses/LM/BackwardLMState.cpp @@ -25,10 +25,16 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -int BackwardLMState::Compare(const FFState &o) const +size_t BackwardLMState::hash() const +{ + size_t ret = hash_value(state.left); + return ret; +} +bool BackwardLMState::operator==(const FFState& o) const { const BackwardLMState &other = static_cast<const BackwardLMState &>(o); - return state.left.Compare(other.state.left); + bool ret = state.left == other.state.left; + return ret; } } diff --git a/moses/LM/BackwardLMState.h b/moses/LM/BackwardLMState.h index 09a768462..119452ff0 100644 --- a/moses/LM/BackwardLMState.h +++ b/moses/LM/BackwardLMState.h @@ -47,13 +47,8 @@ class BackwardLMState : public FFState public: - /* - int Compare(const FFState &o) const { - const BackwardLMState &other = static_cast<const BackwardLMState &>(o); - return state.left.Compare(other.state.left); - } - */ - int Compare(const FFState &o) const; + size_t hash() const; + virtual bool operator==(const FFState& other) const; // Allow BackwardLanguageModel to access the private members of this class template <class Model> friend class BackwardLanguageModel; diff --git a/moses/LM/BilingualLM.cpp b/moses/LM/BilingualLM.cpp index d881c4616..ad6dbf3d3 100644 --- a/moses/LM/BilingualLM.cpp +++ b/moses/LM/BilingualLM.cpp @@ -7,15 +7,6 @@ using namespace std; namespace Moses { -int BilingualLMState::Compare(const FFState& other) const -{ - const BilingualLMState &otherState = static_cast<const BilingualLMState&>(other); - - if (m_hash == otherState.m_hash) - return 0; - return (m_hash < otherState.m_hash) ? -1 : +1; -} - //////////////////////////////////////////////////////////////// BilingualLM::BilingualLM(const std::string &line) : StatefulFeatureFunction(1, line), diff --git a/moses/LM/BilingualLM.h b/moses/LM/BilingualLM.h index 67a6c2ea1..15de9d044 100644 --- a/moses/LM/BilingualLM.h +++ b/moses/LM/BilingualLM.h @@ -37,7 +37,14 @@ public: return neuralLM_ids; } - int Compare(const FFState& other) const; + virtual size_t hash() const { + return m_hash; + } + virtual bool operator==(const FFState& other) const { + const BilingualLMState &otherState = static_cast<const BilingualLMState&>(other); + return m_hash == otherState.m_hash; + } + }; class BilingualLM : public StatefulFeatureFunction diff --git a/moses/LM/ChartState.h b/moses/LM/ChartState.h index f5afbca1c..d4a5cfb30 100644 --- a/moses/LM/ChartState.h +++ b/moses/LM/ChartState.h @@ -145,26 +145,46 @@ public: return m_contextSuffix; } - int Compare(const FFState& o) const { + size_t hash() const { + size_t ret; + + // prefix + ret = m_hypo.GetCurrSourceRange().GetStartPos() > 0; + if (m_hypo.GetCurrSourceRange().GetStartPos() > 0) { // not for "<s> ..." + size_t hash = hash_value(GetPrefix()); + boost::hash_combine(ret, hash); + } + + // suffix + size_t inputSize = m_hypo.GetManager().GetSource().GetSize(); + boost::hash_combine(ret, m_hypo.GetCurrSourceRange().GetEndPos() < inputSize - 1); + if (m_hypo.GetCurrSourceRange().GetEndPos() < inputSize - 1) { // not for "... </s>" + size_t hash = m_lmRightContext->hash(); + boost::hash_combine(ret, hash); + } + + return ret; + } + virtual bool operator==(const FFState& o) const { const LanguageModelChartState &other = - dynamic_cast<const LanguageModelChartState &>( o ); + static_cast<const LanguageModelChartState &>( o ); // prefix if (m_hypo.GetCurrSourceRange().GetStartPos() > 0) { // not for "<s> ..." - int ret = GetPrefix().Compare(other.GetPrefix()); - if (ret != 0) - return ret; + bool ret = GetPrefix() == other.GetPrefix(); + if (ret == false) + return false; } // suffix size_t inputSize = m_hypo.GetManager().GetSource().GetSize(); if (m_hypo.GetCurrSourceRange().GetEndPos() < inputSize - 1) { // not for "... </s>" - int ret = other.GetRightContext()->Compare(*m_lmRightContext); - if (ret != 0) - return ret; + bool ret = (*other.GetRightContext()) == (*m_lmRightContext); + return ret; } - return 0; + return true; } + }; } // namespace diff --git a/moses/LM/DALMWrapper.cpp b/moses/LM/DALMWrapper.cpp index 638060b0e..d10f46ebd 100644 --- a/moses/LM/DALMWrapper.cpp +++ b/moses/LM/DALMWrapper.cpp @@ -68,6 +68,13 @@ public: else return state.compare(o.state); } + virtual size_t hash() const { + UTIL_THROW2("TODO:Haven't figure this out yet"); + } + virtual bool operator==(const FFState& other) const { + UTIL_THROW2("TODO:Haven't figure this out yet"); + } + DALM::State &get_state() { return state; } @@ -178,6 +185,14 @@ public: if(rightContext.get_count() > o.rightContext.get_count()) return 1; return rightContext.compare(o.rightContext); } + + virtual size_t hash() const { + UTIL_THROW2("TODO:Haven't figure this out yet"); + } + virtual bool operator==(const FFState& other) const { + UTIL_THROW2("TODO:Haven't figure this out yet"); + } + }; LanguageModelDALM::LanguageModelDALM(const std::string &line) diff --git a/moses/LM/IRST.cpp b/moses/LM/IRST.cpp index a9cefee28..a26abe03b 100644 --- a/moses/LM/IRST.cpp +++ b/moses/LM/IRST.cpp @@ -270,7 +270,9 @@ void LanguageModelIRST::CalcScoreWithContext(ttasksptr const& ttasks, const Phra if ( !phrase.GetSize() ) return; //get the context_weight map here - std::map<std::string, float> context_weight = ttasks->GetContextWeights(); + SPTR<std::map<std::string, float> const> context_weight = ttasks->GetContextWeights(); + UTIL_THROW_IF2(!context_weight, "No context weights available"); + int _min = min(m_lmtb_size - 1, (int) phrase.GetSize()); @@ -313,7 +315,8 @@ FFState* LanguageModelIRST::EvaluateWhenAppliedWithContext(ttasksptr const& ttas } //get the context_weight map here - std::map<std::string, float> context_weight = ttasks->GetContextWeights(); + SPTR<std::map<std::string, float> const> context_weight = ttasks->GetContextWeights(); + UTIL_THROW_IF2(!context_weight, "No context weights available"); //[begin, end) in STL-like fashion. const int begin = (const int) hypo.GetCurrTargetWordsRange().GetStartPos(); diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index e86275050..8309bda20 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "lm/model.hh" #include "util/exception.hh" #include "util/tokenize_piece.hh" +#include "util/string_stream.hh" #include "Ken.h" #include "Base.h" @@ -55,12 +56,16 @@ namespace struct KenLMState : public FFState { lm::ngram::State state; - int Compare(const FFState &o) const { + virtual size_t hash() const { + size_t ret = hash_value(state); + return ret; + } + virtual bool operator==(const FFState& o) const { const KenLMState &other = static_cast<const KenLMState &>(o); - if (state.length < other.state.length) return -1; - if (state.length > other.state.length) return 1; - return std::memcmp(state.words, other.state.words, sizeof(lm::WordIndex) * state.length); + bool ret = state == other.state; + return ret; } + }; ///* @@ -301,9 +306,13 @@ public: return m_state; } - int Compare(const FFState& o) const { - const LanguageModelChartStateKenLM &other = static_cast<const LanguageModelChartStateKenLM&>(o); - int ret = m_state.Compare(other.m_state); + size_t hash() const { + size_t ret = hash_value(m_state); + return ret; + } + virtual bool operator==(const FFState& o) const { + const LanguageModelChartStateKenLM &other = static_cast<const LanguageModelChartStateKenLM &>(o); + bool ret = m_state == other.m_state; return ret; } @@ -383,7 +392,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con } else if (word.IsNonTerminal()) { // Non-terminal is first so we can copy instead of rescoring. const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]]; - const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->state[featureID])->GetChartState(); + const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState(); float prob = UntransformLMScore( pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]); ruleScore.BeginNonTerminal(prevState, prob); @@ -395,7 +404,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con const Word &word = target.GetWord(phrasePos); if (word.IsNonTerminal()) { const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]]; - const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->state[featureID])->GetChartState(); + const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState(); float prob = UntransformLMScore( pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]); ruleScore.NonTerminal(prevState, prob); @@ -466,7 +475,7 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig) util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' '); ++argument; // KENLM - stringstream line; + util::StringStream line; line << "KENLM"; for (; argument; ++argument) { diff --git a/moses/LM/PointerState.h b/moses/LM/PointerState.h index c6c425198..8bc615c40 100644 --- a/moses/LM/PointerState.h +++ b/moses/LM/PointerState.h @@ -10,12 +10,14 @@ struct PointerState : public FFState { PointerState(const void* lms) { lmstate = lms; } - int Compare(const FFState& o) const { - const PointerState& other = static_cast<const PointerState&>(o); - if (other.lmstate > lmstate) return 1; - else if (other.lmstate < lmstate) return -1; - return 0; + virtual size_t hash() const { + return (size_t) lmstate; } + virtual bool operator==(const FFState& other) const { + const PointerState& o = static_cast<const PointerState&>(other); + return lmstate == o.lmstate; + } + }; } // namespace diff --git a/moses/LM/Remote.cpp b/moses/LM/Remote.cpp index 33946442a..4214c3d63 100644 --- a/moses/LM/Remote.cpp +++ b/moses/LM/Remote.cpp @@ -5,6 +5,7 @@ #include <sys/types.h> #include "Remote.h" #include "moses/Factor.h" +#include "util/string_stream.hh" #if !defined(_WIN32) && !defined(_WIN64) #include <arpa/inet.h> @@ -96,7 +97,7 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa cur->boState = *reinterpret_cast<const State*>(&m_curId); ++m_curId; - std::ostringstream os; + util::StringStream os; os << "prob "; if (event_word == NULL) { os << "</s>"; @@ -111,9 +112,8 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa os << ' ' << f->GetString(); } } - os << std::endl; - std::string out = os.str(); - write(sock, out.c_str(), out.size()); + os << "\n"; + write(sock, os.str().c_str(), os.str().size()); char res[6]; int r = read(sock, res, 6); int errors = 0; diff --git a/moses/Manager.cpp b/moses/Manager.cpp index f5168f244..8112193ef 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -56,6 +56,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "util/exception.hh" #include "util/random.hh" +#include "util/string_stream.hh" using namespace std; @@ -520,7 +521,7 @@ void Manager::OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothe const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); for (size_t i = 0; i < statefulFFs.size(); ++i) { const StatefulFeatureFunction *ff = statefulFFs[i]; - const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + const LanguageModel *lm = static_cast<const LanguageModel*>(ff); vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lm); @@ -1971,7 +1972,7 @@ void Manager::OutputSearchGraphSLF() const // Output search graph in HTK standard lattice format (SLF) bool slf = staticData.GetOutputSearchGraphSLF(); if (slf) { - stringstream fileName; + util::StringStream fileName; string dir; staticData.GetParameter().SetParameter<string>(dir, "output-search-graph-slf", ""); diff --git a/moses/MockHypothesis.cpp b/moses/MockHypothesis.cpp index a087400be..32bb200f0 100644 --- a/moses/MockHypothesis.cpp +++ b/moses/MockHypothesis.cpp @@ -17,12 +17,11 @@ License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ +#include <boost/test/unit_test.hpp> #include "MockHypothesis.h" #include "TranslationOption.h" #include "TranslationTask.h" - -#include <boost/test/unit_test.hpp> - +#include "Bitmaps.h" using namespace Moses; using namespace std; @@ -45,8 +44,11 @@ MockHypothesisGuard m_manager.reset(new Manager(m_ttask)); //Initial empty hypothesis + Bitmaps bitmaps(m_sentence.get()->GetSize(), m_sentence.get()->m_sourceCompleted); m_manager->ResetSentenceStats(*m_sentence); - m_hypothesis = Hypothesis::Create(*m_manager, *m_sentence, m_initialTransOpt); + + const WordsBitmap &initBitmap = bitmaps.GetInitialBitmap(); + m_hypothesis = new Hypothesis(*m_manager, *m_sentence, m_initialTransOpt, initBitmap); //create the chain vector<Alignment>::const_iterator ai = alignments.begin(); @@ -54,12 +56,15 @@ MockHypothesisGuard for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) { Hypothesis* prevHypo = m_hypothesis; WordsRange wordsRange(ai->first,ai->second); + const WordsBitmap &newBitmap = bitmaps.GetBitmap(prevHypo->GetWordsBitmap(), wordsRange); + m_targetPhrases.push_back(TargetPhrase(NULL)); // m_targetPhrases.back().CreateFromString(Input, factors, *ti, "|", NULL); m_targetPhrases.back().CreateFromString(Input, factors, *ti, NULL); m_toptions.push_back(new TranslationOption (wordsRange,m_targetPhrases.back())); - m_hypothesis = Hypothesis::Create(*prevHypo,*m_toptions.back()); + m_hypothesis = new Hypothesis(*prevHypo, *m_toptions.back(), newBitmap); + } diff --git a/moses/PDTAimp.cpp b/moses/PDTAimp.cpp index 6770a5c17..c4bd2c338 100644 --- a/moses/PDTAimp.cpp +++ b/moses/PDTAimp.cpp @@ -11,7 +11,7 @@ PDTAimp::PDTAimp(PhraseDictionaryTreeAdaptor *p) distinctE(0) { m_numInputScores = 0; - m_inputFeature = &InputFeature::Instance(); + m_inputFeature = InputFeature::InstancePtr(); if (m_inputFeature) { const PhraseDictionary *firstPt = PhraseDictionary::GetColl()[0]; @@ -63,27 +63,29 @@ void PDTAimp::CleanUp() { assert(m_dict); m_dict->FreeMemory(); - for(size_t i=0; i<m_tgtColls.size(); ++i) delete m_tgtColls[i]; + // for(size_t i=0; i<m_tgtColls.size(); ++i) m_tgtColls[i].reset(); m_tgtColls.clear(); m_cache.clear(); m_rangeCache.clear(); uniqSrcPhr.clear(); } -TargetPhraseCollectionWithSourcePhrase const* +TargetPhraseCollectionWithSourcePhrase::shared_ptr PDTAimp::GetTargetPhraseCollection(Phrase const &src) const { assert(m_dict); - if(src.GetSize()==0) return 0; + + TargetPhraseCollectionWithSourcePhrase::shared_ptr ret; + if(src.GetSize()==0) return ret; std::pair<MapSrc2Tgt::iterator,bool> piter; if(useCache) { - piter=m_cache.insert(std::make_pair(src,static_cast<TargetPhraseCollectionWithSourcePhrase const*>(0))); + piter=m_cache.insert(std::make_pair(src, ret)); if(!piter.second) return piter.first->second; } else if (m_cache.size()) { MapSrc2Tgt::const_iterator i=m_cache.find(src); - return (i!=m_cache.end() ? i->second : 0); + return (i!=m_cache.end() ? i->second : ret); } std::vector<std::string> srcString(src.GetSize()); @@ -97,7 +99,7 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const std::vector<std::string> wacands; m_dict->GetTargetCandidates(srcString,cands,wacands); if(cands.empty()) { - return 0; + return ret; } //TODO: Multiple models broken here @@ -140,16 +142,14 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const sourcePhrases.push_back(src); } - TargetPhraseCollectionWithSourcePhrase *rv; - rv=PruneTargetCandidates(tCands,costs, sourcePhrases); - if(rv->IsEmpty()) { - delete rv; - return 0; + ret = PruneTargetCandidates(tCands,costs, sourcePhrases); + if(ret->IsEmpty()) { + ret.reset(); } else { - if(useCache) piter.first->second=rv; - m_tgtColls.push_back(rv); - return rv; + if(useCache) piter.first->second = ret; + m_tgtColls.push_back(ret); } + return ret; } @@ -352,7 +352,8 @@ void PDTAimp::CacheSource(ConfusionNet const& src) pathExplored[len]+=exploredPaths[len]; - m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize(),0)); + // m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize(),0)); + m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize())); for(std::map<Range,E2Costs>::const_iterator i=cov2cand.begin(); i!=cov2cand.end(); ++i) { assert(i->first.first<m_rangeCache.size()); @@ -386,10 +387,11 @@ void PDTAimp::CacheSource(ConfusionNet const& src) //std::cerr << i->first.first << "-" << i->first.second << ": " << targetPhrase << std::endl; } - TargetPhraseCollectionWithSourcePhrase *rv=PruneTargetCandidates(tCands, costs, sourcePhrases); + TargetPhraseCollectionWithSourcePhrase::shared_ptr + rv = PruneTargetCandidates(tCands, costs, sourcePhrases); if(rv->IsEmpty()) - delete rv; + rv.reset(); else { m_rangeCache[i->first.first][i->first.second-1]=rv; m_tgtColls.push_back(rv); @@ -428,7 +430,8 @@ void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, targetPhrase.EvaluateInIsolation(*srcPtr, m_obj->GetFeaturesToApply()); } -TargetPhraseCollectionWithSourcePhrase* PDTAimp::PruneTargetCandidates +TargetPhraseCollectionWithSourcePhrase::shared_ptr +PDTAimp::PruneTargetCandidates (const std::vector<TargetPhrase> & tCands, std::vector<std::pair<float,size_t> >& costs, const std::vector<Phrase> &sourcePhrases) const @@ -437,7 +440,8 @@ TargetPhraseCollectionWithSourcePhrase* PDTAimp::PruneTargetCandidates UTIL_THROW_IF2(tCands.size() != sourcePhrases.size(), "Number of target phrases must equal number of source phrases"); - TargetPhraseCollectionWithSourcePhrase *rv=new TargetPhraseCollectionWithSourcePhrase; + TargetPhraseCollectionWithSourcePhrase::shared_ptr rv; + rv.reset(new TargetPhraseCollectionWithSourcePhrase); // set limit to tableLimit or actual size, whatever is smaller diff --git a/moses/PDTAimp.h b/moses/PDTAimp.h index 01de1e88a..81ce22cfe 100644 --- a/moses/PDTAimp.h +++ b/moses/PDTAimp.h @@ -44,10 +44,10 @@ public: std::vector<FactorType> m_input,m_output; PhraseDictionaryTree *m_dict; const InputFeature *m_inputFeature; - typedef std::vector<TargetPhraseCollectionWithSourcePhrase const*> vTPC; + typedef std::vector<TargetPhraseCollectionWithSourcePhrase::shared_ptr> vTPC; mutable vTPC m_tgtColls; - typedef std::map<Phrase,TargetPhraseCollectionWithSourcePhrase const*> MapSrc2Tgt; + typedef std::map<Phrase,TargetPhraseCollectionWithSourcePhrase::shared_ptr> MapSrc2Tgt; mutable MapSrc2Tgt m_cache; PhraseDictionaryTreeAdaptor *m_obj; int useCache; @@ -69,7 +69,7 @@ public: void CleanUp(); - TargetPhraseCollectionWithSourcePhrase const* + TargetPhraseCollectionWithSourcePhrase::shared_ptr GetTargetPhraseCollection(Phrase const &src) const; void Create(const std::vector<FactorType> &input @@ -121,7 +121,7 @@ public: const std::string *alignmentString, Phrase const* srcPtr=0) const; - TargetPhraseCollectionWithSourcePhrase* PruneTargetCandidates + TargetPhraseCollectionWithSourcePhrase::shared_ptr PruneTargetCandidates (const std::vector<TargetPhrase> & tCands, std::vector<std::pair<float,size_t> >& costs, const std::vector<Phrase> &sourcePhrases) const; diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp index d0b6d6374..f6058cbd8 100644 --- a/moses/Parameter.cpp +++ b/moses/Parameter.cpp @@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "Util.h" #include "InputFileStream.h" #include "StaticData.h" +#include "util/string_stream.hh" #include "util/exception.hh" #include "util/random.hh" #include <boost/program_options.hpp> @@ -116,6 +117,7 @@ Parameter::Parameter() AddParam(cube_opts,"cube-pruning-pop-limit", "cbp", "How many hypotheses should be popped for each stack. (default = 1000)"); AddParam(cube_opts,"cube-pruning-diversity", "cbd", "How many hypotheses should be created for each coverage. (default = 0)"); AddParam(cube_opts,"cube-pruning-lazy-scoring", "cbls", "Don't fully score a hypothesis until it is popped"); + AddParam(cube_opts,"cube-pruning-deterministic-search", "cbds", "Break ties deterministically during search"); /////////////////////////////////////////////////////////////////////////////////////// // minimum bayes risk decoding @@ -700,7 +702,7 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName) size_t currOldInd = 0; for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) { - stringstream ptLine; + util::StringStream ptLine; vector<string> token = Tokenize(translationVector[currDict]); @@ -859,7 +861,7 @@ ConvertWeightArgsDistortion() } SetWeight("LexicalReordering", indTable, weights); - stringstream strme; + util::StringStream strme; strme << "LexicalReordering " << "type=" << toks[1] << " "; @@ -1006,7 +1008,7 @@ ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string } SetWeight(newWeightName, indTable, weights); - stringstream strme; + util::StringStream strme; strme << "Generation " << "input-factor=" << modelToks[0] << " " << "output-factor=" << modelToks[1] << " " diff --git a/moses/Phrase.cpp b/moses/Phrase.cpp index 7a9e847ba..7ab3464df 100644 --- a/moses/Phrase.cpp +++ b/moses/Phrase.cpp @@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "StaticData.h" // GetMaxNumFactors #include "util/string_piece.hh" +#include "util/string_stream.hh" #include "util/tokenize_piece.hh" using namespace std; @@ -117,7 +118,7 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const { bool markUnknown = StaticData::Instance().GetMarkUnknown(); - stringstream strme; + util::StringStream strme; for (size_t pos = 0 ; pos < GetSize() ; pos++) { if (markUnknown && GetWord(pos).IsOOV()) { strme << StaticData::Instance().GetUnknownWordPrefix(); @@ -253,6 +254,35 @@ int Phrase::Compare(const Phrase &other) const return 0; } +size_t Phrase::hash() const +{ + size_t seed = 0; + for (size_t i = 0; i < GetSize(); ++i) { + boost::hash_combine(seed, GetWord(i)); + } + return seed; +} + +bool Phrase::operator== (const Phrase &other) const +{ + size_t thisSize = GetSize() + ,compareSize = other.GetSize(); + if (thisSize != compareSize) { + return false; + } + + for (size_t pos = 0 ; pos < thisSize ; pos++) { + const Word &thisWord = GetWord(pos) + ,&otherWord = other.GetWord(pos); + bool ret = thisWord == otherWord; + if (!ret) { + return false; + } + } + + return true; +} + bool Phrase::Contains(const vector< vector<string> > &subPhraseVector , const vector<FactorType> &inputFactor) const diff --git a/moses/Phrase.h b/moses/Phrase.h index 947e50905..e92d68ac4 100644 --- a/moses/Phrase.h +++ b/moses/Phrase.h @@ -192,8 +192,11 @@ public: return Compare(compare) < 0; } - bool operator== (const Phrase &compare) const { - return Compare(compare) == 0; + size_t hash() const; + + bool operator==(const Phrase &compare) const; + bool operator!=(const Phrase &compare) const { + return ! (*this == compare); } void OnlyTheseFactors(const FactorMask &factors); @@ -202,11 +205,7 @@ public: inline size_t hash_value(const Phrase& phrase) { - size_t seed = 0; - for (size_t i = 0; i < phrase.GetSize(); ++i) { - boost::hash_combine(seed, phrase.GetWord(i)); - } - return seed; + return phrase.hash(); } struct PhrasePtrComparator { diff --git a/moses/ScoreComponentCollection.cpp b/moses/ScoreComponentCollection.cpp index 4d79d5565..d9810224e 100644 --- a/moses/ScoreComponentCollection.cpp +++ b/moses/ScoreComponentCollection.cpp @@ -4,6 +4,7 @@ #include <boost/foreach.hpp> #include <boost/format.hpp> #include "util/exception.hh" +#include "util/string_stream.hh" #include "ScoreComponentCollection.h" #include "StaticData.h" #include "moses/FF/StatelessFeatureFunction.h" @@ -88,9 +89,8 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s { std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { - std::stringstream name; - name << i->first; - if (starts_with(name.str(), prefix)) + const std::string &name = i->first.name(); + if (starts_with(name, prefix)) m_scores[i->first] = i->second * scalar; } } @@ -101,9 +101,8 @@ size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp) std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; size_t weights = 0; for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { - std::stringstream name; - name << i->first; - if (starts_with(name.str(), prefix)) + const std::string &name = i->first.name(); + if (starts_with(name, prefix)) weights++; } return weights; @@ -215,7 +214,7 @@ void ScoreComponentCollection::Save(const string& filename) const { ofstream out(filename.c_str()); if (!out) { - ostringstream msg; + util::StringStream msg; msg << "Unable to open " << filename; throw runtime_error(msg.str()); } diff --git a/moses/SearchCubePruning.cpp b/moses/SearchCubePruning.cpp index 7219a40f7..8e6cf993d 100644 --- a/moses/SearchCubePruning.cpp +++ b/moses/SearchCubePruning.cpp @@ -32,7 +32,14 @@ public: } else if (scoreA > scoreB) { return false; } else { - return A < B; + // Equal scores: break ties by comparing target phrases (if they exist) + boost::shared_ptr<TargetPhrase> phrA = A->Top()->GetTargetPhrase(); + boost::shared_ptr<TargetPhrase> phrB = B->Top()->GetTargetPhrase(); + if (!phrA || !phrB) { + // Fallback: compare pointers, non-deterministic sort + return A < B; + } + return (phrA->Compare(*phrB) < 0); } } }; @@ -42,6 +49,7 @@ SearchCubePruning(Manager& manager, const InputType &source, const TranslationOptionCollection &transOptColl) : Search(manager) , m_source(source) + , m_initBitmap(source.GetSize()) , m_hypoStackColl(source.GetSize() + 1) , m_transOptColl(transOptColl) { @@ -67,7 +75,7 @@ SearchCubePruning::~SearchCubePruning() void SearchCubePruning::Decode() { // initial seed hypothesis: nothing translated, no words produced - Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt); + Hypothesis *hypo = new Hypothesis(m_manager, m_source, m_initialTransOpt, m_initBitmap); HypothesisStackCubePruning &firstStack = *static_cast<HypothesisStackCubePruning*>(m_hypoStackColl.front()); diff --git a/moses/SearchCubePruning.h b/moses/SearchCubePruning.h index 334204004..b30adf772 100644 --- a/moses/SearchCubePruning.h +++ b/moses/SearchCubePruning.h @@ -19,6 +19,7 @@ class SearchCubePruning: public Search { protected: const InputType &m_source; + WordsBitmap m_initBitmap; std::vector < HypothesisStack* > m_hypoStackColl; /**< stacks to store hypotheses (partial translations) */ // no of elements = no of words in source + 1 const TranslationOptionCollection &m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */ diff --git a/moses/SearchNormal.cpp b/moses/SearchNormal.cpp index 7056aeb51..c6d42c666 100644 --- a/moses/SearchNormal.cpp +++ b/moses/SearchNormal.cpp @@ -22,6 +22,7 @@ SearchNormal(Manager& manager, const InputType &source, , m_source(source) , m_hypoStackColl(source.GetSize() + 1) , m_transOptColl(transOptColl) + , m_bitmaps(source.GetSize(), source.m_sourceCompleted) { VERBOSE(1, "Translating: " << m_source << endl); @@ -86,7 +87,9 @@ void SearchNormal::Decode() // SentenceStats &stats = m_manager.GetSentenceStats(); // initial seed hypothesis: nothing translated, no words produced - Hypothesis *hypo = Hypothesis::Create(m_manager, m_source, m_initialTransOpt); + const WordsBitmap &bitmap = m_bitmaps.GetInitialBitmap(); + Hypothesis *hypo = new Hypothesis(m_manager, m_source, m_initialTransOpt, bitmap); + m_hypoStackColl[0]->AddPrune(hypo); // go through each stack @@ -248,23 +251,32 @@ ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos // early discarding: check if hypothesis is too bad to build // this idea is explained in (Moore&Quirk, MT Summit 2007) float expectedScore = 0.0f; + + const WordsBitmap &sourceCompleted = hypothesis.GetWordsBitmap(); + float futureScore = m_transOptColl.GetFutureScore().CalcFutureScore2( sourceCompleted, startPos, endPos ); + if (m_options.search.UseEarlyDiscarding()) { // expected score is based on score of current hypothesis expectedScore = hypothesis.GetScore(); // add new future score estimate - expectedScore += - m_transOptColl.GetFutureScore() - .CalcFutureScore(hypothesis.GetWordsBitmap(), startPos, endPos); + expectedScore += futureScore; } // loop through all translation options const TranslationOptionList* tol = m_transOptColl.GetTranslationOptionList(startPos, endPos); - if (!tol) return; + if (!tol || tol->size() == 0) return; + + // Create new bitmap + const TranslationOption &transOpt = **tol->begin(); + const WordsRange &nextRange = transOpt.GetSourceWordsRange(); + const WordsBitmap &nextBitmap = m_bitmaps.GetBitmap(sourceCompleted, nextRange); + TranslationOptionList::const_iterator iter; for (iter = tol->begin() ; iter != tol->end() ; ++iter) { - ExpandHypothesis(hypothesis, **iter, expectedScore); + const TranslationOption &transOpt = **iter; + ExpandHypothesis(hypothesis, transOpt, expectedScore, futureScore, nextBitmap); } } @@ -277,7 +289,11 @@ ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos * \param expectedScore base score for early discarding * (base hypothesis score plus future score estimation) */ -void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt, float expectedScore) +void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, + const TranslationOption &transOpt, + float expectedScore, + float futureScore, + const WordsBitmap &bitmap) { const StaticData &staticData = StaticData::Instance(); SentenceStats &stats = m_manager.GetSentenceStats(); @@ -288,12 +304,12 @@ void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const Translat IFVERBOSE(2) { stats.StartTimeBuildHyp(); } - newHypo = hypothesis.CreateNext(transOpt); + newHypo = new Hypothesis(hypothesis, transOpt, bitmap); IFVERBOSE(2) { stats.StopTimeBuildHyp(); } if (newHypo==NULL) return; - newHypo->EvaluateWhenApplied(m_transOptColl.GetFutureScore()); + newHypo->EvaluateWhenApplied(futureScore); } else // early discarding: check if hypothesis is too bad to build { @@ -322,7 +338,7 @@ void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const Translat IFVERBOSE(2) { stats.StartTimeBuildHyp(); } - newHypo = hypothesis.CreateNext(transOpt); + newHypo = new Hypothesis(hypothesis, transOpt, bitmap); if (newHypo==NULL) return; IFVERBOSE(2) { stats.StopTimeBuildHyp(); @@ -333,7 +349,7 @@ void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const Translat IFVERBOSE(2) { stats.AddEarlyDiscarded(); } - FREEHYPO( newHypo ); + delete newHypo; return; } diff --git a/moses/SearchNormal.h b/moses/SearchNormal.h index aa005a5e1..9c2e11ad8 100644 --- a/moses/SearchNormal.h +++ b/moses/SearchNormal.h @@ -6,6 +6,7 @@ #include "HypothesisStackNormal.h" #include "TranslationOptionCollection.h" #include "Timer.h" +#include "Bitmaps.h" namespace Moses { @@ -32,6 +33,8 @@ protected: /** pre-computed list of translation options for the phrases in this sentence */ const TranslationOptionCollection &m_transOptColl; + Bitmaps m_bitmaps; + // functions for creating hypotheses virtual bool @@ -44,8 +47,11 @@ protected: ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos); virtual void - ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt, - float expectedScore); + ExpandHypothesis(const Hypothesis &hypothesis, + const TranslationOption &transOpt, + float expectedScore, + float futureScore, + const WordsBitmap &bitmap); public: SearchNormal(Manager& manager, const InputType &source, const TranslationOptionCollection &transOptColl); diff --git a/moses/SquareMatrix.cpp b/moses/SquareMatrix.cpp index 276d10053..0580ba096 100644 --- a/moses/SquareMatrix.cpp +++ b/moses/SquareMatrix.cpp @@ -76,7 +76,7 @@ float SquareMatrix::CalcFutureScore( WordsBitmap const &bitmap ) const * /param endPos end of the span that is added to the coverage */ -float SquareMatrix::CalcFutureScore( WordsBitmap const &bitmap, size_t startPos, size_t endPos ) const +float SquareMatrix::CalcFutureScore2( WordsBitmap const &bitmap, size_t startPos, size_t endPos ) const { const size_t notInGap= numeric_limits<size_t>::max(); float futureScore = 0.0f; diff --git a/moses/SquareMatrix.h b/moses/SquareMatrix.h index ed9a5e8fa..ebfea1ffb 100644 --- a/moses/SquareMatrix.h +++ b/moses/SquareMatrix.h @@ -62,7 +62,7 @@ public: m_array[startPos * m_size + endPos] = value; } float CalcFutureScore( WordsBitmap const& ) const; - float CalcFutureScore( WordsBitmap const&, size_t startPos, size_t endPos ) const; + float CalcFutureScore2( WordsBitmap const&, size_t startPos, size_t endPos ) const; TO_STRING(); }; diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 53b60e850..341e4d0cc 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -640,7 +640,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const switch (decodeType) { case Translate: if(index>=pts.size()) { - stringstream strme; + util::StringStream strme; strme << "No phrase dictionary with index " << index << " available!"; UTIL_THROW(util::Exception, strme.str()); @@ -649,7 +649,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const break; case Generate: if(index>=gens.size()) { - stringstream strme; + util::StringStream strme; strme << "No generation dictionary with index " << index << " available!"; UTIL_THROW(util::Exception, strme.str()); diff --git a/moses/Syntax/Cube.cpp b/moses/Syntax/Cube.cpp index 6cbf72903..0493c448e 100644 --- a/moses/Syntax/Cube.cpp +++ b/moses/Syntax/Cube.cpp @@ -93,7 +93,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates) SVertex *head = new SVertex(); head->best = hyperedge; head->pvertex = 0; // FIXME??? - head->state.resize( + head->states.resize( StatefulFeatureFunction::GetStatefulFeatureFunctions().size()); hyperedge->head = head; @@ -131,7 +131,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates) StatefulFeatureFunction::GetStatefulFeatureFunctions(); for (unsigned i = 0; i < ffs.size(); ++i) { if (!staticData.IsFeatureFunctionIgnored(*ffs[i])) { - head->state[i] = + head->states[i] = ffs[i]->EvaluateWhenApplied(*hyperedge, i, &hyperedge->label.scoreBreakdown); } diff --git a/moses/Syntax/F2S/GlueRuleSynthesizer.cpp b/moses/Syntax/F2S/GlueRuleSynthesizer.cpp index 09423f5d3..3ba5a26d3 100644 --- a/moses/Syntax/F2S/GlueRuleSynthesizer.cpp +++ b/moses/Syntax/F2S/GlueRuleSynthesizer.cpp @@ -4,6 +4,7 @@ #include "moses/FF/UnknownWordPenaltyProducer.h" #include "moses/StaticData.h" +#include "util/string_stream.hh" namespace Moses { @@ -27,9 +28,9 @@ void GlueRuleSynthesizer::SynthesizeRule(const Forest::Hyperedge &e) HyperPath source; SynthesizeHyperPath(e, source); TargetPhrase *tp = SynthesizeTargetPhrase(e); - TargetPhraseCollection &tpc = GetOrCreateTargetPhraseCollection(m_hyperTree, - source); - tpc.Add(tp); + TargetPhraseCollection::shared_ptr tpc + = GetOrCreateTargetPhraseCollection(m_hyperTree, source); + tpc->Add(tp); } void GlueRuleSynthesizer::SynthesizeHyperPath(const Forest::Hyperedge &e, @@ -55,7 +56,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase( TargetPhrase *targetPhrase = new TargetPhrase(); - std::ostringstream alignmentSS; + util::StringStream alignmentSS; for (std::size_t i = 0; i < e.tail.size(); ++i) { const Word &symbol = e.tail[i]->pvertex.symbol; if (symbol.IsNonTerminal()) { diff --git a/moses/Syntax/F2S/HyperTree.cpp b/moses/Syntax/F2S/HyperTree.cpp index a7d77eb4c..681ec561c 100644 --- a/moses/Syntax/F2S/HyperTree.cpp +++ b/moses/Syntax/F2S/HyperTree.cpp @@ -14,7 +14,7 @@ void HyperTree::Node::Prune(std::size_t tableLimit) p->second.Prune(tableLimit); } // Prune TargetPhraseCollection at this node. - m_targetPhraseCollection.Prune(true, tableLimit); + m_targetPhraseCollection->Prune(true, tableLimit); } void HyperTree::Node::Sort(std::size_t tableLimit) @@ -24,7 +24,7 @@ void HyperTree::Node::Sort(std::size_t tableLimit) p->second.Sort(tableLimit); } // Sort TargetPhraseCollection at this node. - m_targetPhraseCollection.Sort(true, tableLimit); + m_targetPhraseCollection->Sort(true, tableLimit); } HyperTree::Node *HyperTree::Node::GetOrCreateChild( @@ -40,7 +40,7 @@ const HyperTree::Node *HyperTree::Node::GetChild( return (p == m_map.end()) ? NULL : &p->second; } -TargetPhraseCollection &HyperTree::GetOrCreateTargetPhraseCollection( +TargetPhraseCollection::shared_ptr HyperTree::GetOrCreateTargetPhraseCollection( const HyperPath &hyperPath) { Node &node = GetOrCreateNode(hyperPath); diff --git a/moses/Syntax/F2S/HyperTree.h b/moses/Syntax/F2S/HyperTree.h index e32bfebbe..8e33451a8 100644 --- a/moses/Syntax/F2S/HyperTree.h +++ b/moses/Syntax/F2S/HyperTree.h @@ -37,7 +37,7 @@ public: } bool HasRules() const { - return !m_targetPhraseCollection.IsEmpty(); + return !m_targetPhraseCollection->IsEmpty(); } void Prune(std::size_t tableLimit); @@ -47,11 +47,13 @@ public: const Node *GetChild(const HyperPath::NodeSeq &) const; - const TargetPhraseCollection &GetTargetPhraseCollection() const { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() const { return m_targetPhraseCollection; } - TargetPhraseCollection &GetTargetPhraseCollection() { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() { return m_targetPhraseCollection; } @@ -59,12 +61,14 @@ public: return m_map; } + Node() : m_targetPhraseCollection(new TargetPhraseCollection) { } + private: Map m_map; - TargetPhraseCollection m_targetPhraseCollection; + TargetPhraseCollection::shared_ptr m_targetPhraseCollection; }; - HyperTree(const RuleTableFF *ff) : RuleTable(ff) {} + HyperTree(const RuleTableFF *ff) : RuleTable(ff) { } const Node &GetRootNode() const { return m_root; @@ -73,7 +77,8 @@ public: private: friend class HyperTreeCreator; - TargetPhraseCollection &GetOrCreateTargetPhraseCollection(const HyperPath &); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const HyperPath &); Node &GetOrCreateNode(const HyperPath &); diff --git a/moses/Syntax/F2S/HyperTreeCreator.h b/moses/Syntax/F2S/HyperTreeCreator.h index 04dbaa6b1..a5111b90e 100644 --- a/moses/Syntax/F2S/HyperTreeCreator.h +++ b/moses/Syntax/F2S/HyperTreeCreator.h @@ -21,7 +21,7 @@ protected: // Provide access to HyperTree's private GetOrCreateTargetPhraseCollection // function. - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( + TargetPhraseCollection::shared_ptr GetOrCreateTargetPhraseCollection( HyperTree &trie, const HyperPath &fragment) { return trie.GetOrCreateTargetPhraseCollection(fragment); } diff --git a/moses/Syntax/F2S/HyperTreeLoader.cpp b/moses/Syntax/F2S/HyperTreeLoader.cpp index 21d5b0447..aefddbc2d 100644 --- a/moses/Syntax/F2S/HyperTreeLoader.cpp +++ b/moses/Syntax/F2S/HyperTreeLoader.cpp @@ -130,9 +130,9 @@ bool HyperTreeLoader::Load(const std::vector<FactorType> &input, ff.GetFeaturesToApply()); // Add rule to trie. - TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection( - trie, sourceFragment); - phraseColl.Add(targetPhrase); + TargetPhraseCollection::shared_ptr phraseColl + = GetOrCreateTargetPhraseCollection(trie, sourceFragment); + phraseColl->Add(targetPhrase); count++; } diff --git a/moses/Syntax/F2S/Manager-inl.h b/moses/Syntax/F2S/Manager-inl.h index 5c1b41295..19446c5cd 100644 --- a/moses/Syntax/F2S/Manager-inl.h +++ b/moses/Syntax/F2S/Manager-inl.h @@ -285,7 +285,7 @@ void Manager<RuleMatcher>::RecombineAndSort( // head pointers are updated to point to the vertex instances in the map and // any 'duplicate' vertices are deleted. // TODO Set? - typedef std::map<SVertex *, SVertex *, SVertexRecombinationOrderer> Map; + typedef boost::unordered_map<SVertex *, SVertex *, SVertexRecombinationUnordered, SVertexRecombinationUnordered> Map; Map map; for (std::vector<SHyperedge*>::const_iterator p = buffer.begin(); p != buffer.end(); ++p) { diff --git a/moses/Syntax/F2S/Manager.h b/moses/Syntax/F2S/Manager.h index bcf1ff2bd..7514338f7 100644 --- a/moses/Syntax/F2S/Manager.h +++ b/moses/Syntax/F2S/Manager.h @@ -1,10 +1,10 @@ #pragma once -#include <set> #include <vector> #include <boost/shared_ptr.hpp> #include <boost/unordered_map.hpp> +#include <boost/unordered_set.hpp> #include "moses/InputType.h" #include "moses/Syntax/KBestExtractor.h" diff --git a/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h b/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h index 74f2347a6..36dfeced2 100644 --- a/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h +++ b/moses/Syntax/F2S/RuleMatcherHyperTree-inl.h @@ -51,8 +51,8 @@ void RuleMatcherHyperTree<Callback>::EnumerateHyperedges( m_hyperedge.label.inputWeight += (*p)->weight; } // Set the output hyperedge label's translation set pointer. - m_hyperedge.label.translations = - &(item.trieNode->GetTargetPhraseCollection()); + m_hyperedge.label.translations + = item.trieNode->GetTargetPhraseCollection(); // Pass the output hyperedge to the callback. callback(m_hyperedge); } diff --git a/moses/Syntax/Manager.cpp b/moses/Syntax/Manager.cpp index 5dd4c3b64..10b0d25c9 100644 --- a/moses/Syntax/Manager.cpp +++ b/moses/Syntax/Manager.cpp @@ -65,7 +65,7 @@ void Manager::OutputUnknowns(OutputCollector *collector) const long translationId = m_source.GetTranslationId(); std::ostringstream out; - for (std::set<Moses::Word>::const_iterator p = m_oovs.begin(); + for (boost::unordered_set<Moses::Word>::const_iterator p = m_oovs.begin(); p != m_oovs.end(); ++p) { out << *p; } diff --git a/moses/Syntax/Manager.h b/moses/Syntax/Manager.h index d62e2f72a..a80c756a8 100644 --- a/moses/Syntax/Manager.h +++ b/moses/Syntax/Manager.h @@ -1,5 +1,6 @@ #pragma once +#include <boost/unordered_set.hpp> #include "moses/InputType.h" #include "moses/BaseManager.h" @@ -50,7 +51,7 @@ public: virtual const SHyperedge *GetBestSHyperedge() const = 0; protected: - std::set<Word> m_oovs; + boost::unordered_set<Word> m_oovs; private: // Syntax-specific helper functions used to implement OutputNBest. diff --git a/moses/Syntax/PLabel.h b/moses/Syntax/PLabel.h index 4537b86bb..c45f40dfd 100644 --- a/moses/Syntax/PLabel.h +++ b/moses/Syntax/PLabel.h @@ -9,7 +9,7 @@ namespace Syntax struct PLabel { float inputWeight; - const TargetPhraseCollection *translations; + TargetPhraseCollection::shared_ptr translations; }; } // Syntax diff --git a/moses/Syntax/S2T/Manager-inl.h b/moses/Syntax/S2T/Manager-inl.h index 4963ec788..422f14915 100644 --- a/moses/Syntax/S2T/Manager-inl.h +++ b/moses/Syntax/S2T/Manager-inl.h @@ -108,7 +108,7 @@ void Manager<Parser>::InitializeParsers(PChart &pchart, // Find the set of OOVs for this input. This function assumes that the // PChart argument has already been initialized from the input. template<typename Parser> -void Manager<Parser>::FindOovs(const PChart &pchart, std::set<Word> &oovs, +void Manager<Parser>::FindOovs(const PChart &pchart, boost::unordered_set<Word> &oovs, std::size_t maxOovWidth) { // Get the set of RuleTries. @@ -349,7 +349,7 @@ void Manager<Parser>::RecombineAndSort(const std::vector<SHyperedge*> &buffer, // head pointers are updated to point to the vertex instances in the map and // any 'duplicate' vertices are deleted. // TODO Set? - typedef std::map<SVertex *, SVertex *, SVertexRecombinationOrderer> Map; + typedef boost::unordered_map<SVertex *, SVertex *, SVertexRecombinationUnordered, SVertexRecombinationUnordered> Map; Map map; for (std::vector<SHyperedge*>::const_iterator p = buffer.begin(); p != buffer.end(); ++p) { diff --git a/moses/Syntax/S2T/Manager.h b/moses/Syntax/S2T/Manager.h index 711d6f9d8..b0e6555cf 100644 --- a/moses/Syntax/S2T/Manager.h +++ b/moses/Syntax/S2T/Manager.h @@ -45,7 +45,7 @@ public: void OutputDetailedTranslationReport(OutputCollector *collector) const; private: - void FindOovs(const PChart &, std::set<Word> &, std::size_t); + void FindOovs(const PChart &, boost::unordered_set<Word> &, std::size_t); void InitializeCharts(); diff --git a/moses/Syntax/S2T/OovHandler-inl.h b/moses/Syntax/S2T/OovHandler-inl.h index 76eed861e..9f26563d5 100644 --- a/moses/Syntax/S2T/OovHandler-inl.h +++ b/moses/Syntax/S2T/OovHandler-inl.h @@ -32,9 +32,10 @@ boost::shared_ptr<RuleTrie> OovHandler<RuleTrie>::SynthesizeRuleTrie( // TODO Check ownership and fix any leaks. Word *tgtLHS = SynthesizeTargetLhs(targetLhsStr); TargetPhrase *tp = SynthesizeTargetPhrase(oov, *srcPhrase, *tgtLHS, prob); - TargetPhraseCollection &tpc = GetOrCreateTargetPhraseCollection( - *trie, *srcPhrase, *tp, NULL); // TODO Check NULL is valid argument - tpc.Add(tp); + TargetPhraseCollection::shared_ptr tpc; + tpc= GetOrCreateTargetPhraseCollection(*trie, *srcPhrase, *tp, NULL); + // TODO Check NULL is valid argument + tpc->Add(tp); } } diff --git a/moses/Syntax/S2T/Parsers/RecursiveCYKPlusParser/RecursiveCYKPlusParser-inl.h b/moses/Syntax/S2T/Parsers/RecursiveCYKPlusParser/RecursiveCYKPlusParser-inl.h index a84e16a54..c8f57c4d1 100644 --- a/moses/Syntax/S2T/Parsers/RecursiveCYKPlusParser/RecursiveCYKPlusParser-inl.h +++ b/moses/Syntax/S2T/Parsers/RecursiveCYKPlusParser/RecursiveCYKPlusParser-inl.h @@ -132,9 +132,9 @@ void RecursiveCYKPlusParser<Callback>::AddAndExtend( m_hyperedge.tail.push_back(const_cast<PVertex *>(&vertex)); // Add target phrase collection (except if rule is empty or unary). - const TargetPhraseCollection &tpc = node.GetTargetPhraseCollection(); - if (!tpc.IsEmpty() && !IsNonLexicalUnary(m_hyperedge)) { - m_hyperedge.label.translations = &tpc; + TargetPhraseCollection::shared_ptr tpc = node.GetTargetPhraseCollection(); + if (!tpc->IsEmpty() && !IsNonLexicalUnary(m_hyperedge)) { + m_hyperedge.label.translations = tpc; (*m_callback)(m_hyperedge, end); } diff --git a/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h b/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h index 24135c734..da81a5606 100644 --- a/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h +++ b/moses/Syntax/S2T/Parsers/Scope3Parser/Parser-inl.h @@ -38,8 +38,8 @@ Scope3Parser<Callback>::~Scope3Parser() } template<typename Callback> -void Scope3Parser<Callback>::EnumerateHyperedges(const WordsRange &range, - Callback &callback) +void Scope3Parser<Callback>:: +EnumerateHyperedges(const WordsRange &range, Callback &callback) { const std::size_t start = range.GetStartPos(); const std::size_t end = range.GetEndPos(); @@ -64,8 +64,7 @@ void Scope3Parser<Callback>::EnumerateHyperedges(const WordsRange &range, // Ask the grammar for the mapping from label sequences to target phrase // collections for this pattern. - const RuleTrie::Node::LabelMap &labelMap = - patNode->m_node->GetLabelMap(); + const RuleTrie::Node::LabelMap &labelMap = patNode->m_node->GetLabelMap(); // For each label sequence, search the lattice for the set of PHyperedge // tails. @@ -73,7 +72,7 @@ void Scope3Parser<Callback>::EnumerateHyperedges(const WordsRange &range, RuleTrie::Node::LabelMap::const_iterator q = labelMap.begin(); for (; q != labelMap.end(); ++q) { const std::vector<int> &labelSeq = q->first; - const TargetPhraseCollection &tpc = q->second; + TargetPhraseCollection::shared_ptr tpc = q->second; // For many label sequences there won't be any corresponding paths through // the lattice. As an optimisation, we use m_quickCheckTable to test // for this and we don't begin a search if there are no paths to find. diff --git a/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h b/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h index 4f815c78d..5ee32553e 100644 --- a/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h +++ b/moses/Syntax/S2T/Parsers/Scope3Parser/TailLatticeSearcher.h @@ -6,7 +6,7 @@ #include "moses/Syntax/PHyperedge.h" #include "TailLattice.h" - +#include "moses/TargetPhraseCollection.h" namespace Moses { namespace Syntax @@ -25,13 +25,14 @@ public: , m_key(key) , m_ranges(ranges) {} - void Search(const std::vector<int> &labels, const TargetPhraseCollection &tpc, + void Search(const std::vector<int> &labels, + const TargetPhraseCollection::shared_ptr tpc, Callback &callback) { m_labels = &labels; m_matchCB = &callback; m_hyperedge.head = 0; m_hyperedge.tail.clear(); - m_hyperedge.label.translations = &tpc; + m_hyperedge.label.translations = tpc; SearchInner(0, 0, 0); } diff --git a/moses/Syntax/S2T/RuleTrie.h b/moses/Syntax/S2T/RuleTrie.h index 27b0bc838..eee6edd5b 100644 --- a/moses/Syntax/S2T/RuleTrie.h +++ b/moses/Syntax/S2T/RuleTrie.h @@ -28,9 +28,10 @@ public: private: friend class RuleTrieCreator; - virtual TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, - const Word *sourceLHS) = 0; + virtual TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) = 0; virtual void SortAndPrune(std::size_t) = 0; }; diff --git a/moses/Syntax/S2T/RuleTrieCYKPlus.cpp b/moses/Syntax/S2T/RuleTrieCYKPlus.cpp index 05f8758e9..68da5f5b7 100644 --- a/moses/Syntax/S2T/RuleTrieCYKPlus.cpp +++ b/moses/Syntax/S2T/RuleTrieCYKPlus.cpp @@ -33,7 +33,7 @@ void RuleTrieCYKPlus::Node::Prune(std::size_t tableLimit) } // prune TargetPhraseCollection in this node - m_targetPhraseCollection.Prune(true, tableLimit); + m_targetPhraseCollection->Prune(true, tableLimit); } void RuleTrieCYKPlus::Node::Sort(std::size_t tableLimit) @@ -49,7 +49,7 @@ void RuleTrieCYKPlus::Node::Sort(std::size_t tableLimit) } // prune TargetPhraseCollection in this node - m_targetPhraseCollection.Sort(true, tableLimit); + m_targetPhraseCollection->Sort(true, tableLimit); } RuleTrieCYKPlus::Node *RuleTrieCYKPlus::Node::GetOrCreateChild( @@ -86,8 +86,11 @@ const RuleTrieCYKPlus::Node *RuleTrieCYKPlus::Node::GetNonTerminalChild( return (p == m_nonTermMap.end()) ? NULL : &p->second; } -TargetPhraseCollection &RuleTrieCYKPlus::GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS) +TargetPhraseCollection::shared_ptr +RuleTrieCYKPlus:: +GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) { Node &currNode = GetOrCreateNode(source, target, sourceLHS); return currNode.GetTargetPhraseCollection(); diff --git a/moses/Syntax/S2T/RuleTrieCYKPlus.h b/moses/Syntax/S2T/RuleTrieCYKPlus.h index 11cf4c199..31880e0ed 100644 --- a/moses/Syntax/S2T/RuleTrieCYKPlus.h +++ b/moses/Syntax/S2T/RuleTrieCYKPlus.h @@ -38,7 +38,7 @@ public: } bool HasRules() const { - return !m_targetPhraseCollection.IsEmpty(); + return !m_targetPhraseCollection->IsEmpty(); } void Prune(std::size_t tableLimit); @@ -50,11 +50,13 @@ public: const Node *GetChild(const Word &sourceTerm) const; const Node *GetNonTerminalChild(const Word &targetNonTerm) const; - const TargetPhraseCollection &GetTargetPhraseCollection() const { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() const { return m_targetPhraseCollection; } - TargetPhraseCollection &GetTargetPhraseCollection() { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() { return m_targetPhraseCollection; } @@ -66,10 +68,12 @@ public: return m_nonTermMap; } + Node() : m_targetPhraseCollection(new TargetPhraseCollection) {} + private: SymbolMap m_sourceTermMap; SymbolMap m_nonTermMap; - TargetPhraseCollection m_targetPhraseCollection; + TargetPhraseCollection::shared_ptr m_targetPhraseCollection; }; RuleTrieCYKPlus(const RuleTableFF *ff) : RuleTrie(ff) {} @@ -81,8 +85,9 @@ public: bool HasPreterminalRule(const Word &) const; private: - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection + (const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); Node &GetOrCreateNode(const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); diff --git a/moses/Syntax/S2T/RuleTrieCreator.h b/moses/Syntax/S2T/RuleTrieCreator.h index e49a2cbde..84a4a1636 100644 --- a/moses/Syntax/S2T/RuleTrieCreator.h +++ b/moses/Syntax/S2T/RuleTrieCreator.h @@ -21,8 +21,9 @@ protected: // Provide access to RuleTrie's private GetOrCreateTargetPhraseCollection // function. - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - RuleTrie &trie, const Phrase &source, const TargetPhrase &target, + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection + ( RuleTrie &trie, const Phrase &source, const TargetPhrase &target, const Word *sourceLHS) { return trie.GetOrCreateTargetPhraseCollection(source, target, sourceLHS); } diff --git a/moses/Syntax/S2T/RuleTrieLoader.cpp b/moses/Syntax/S2T/RuleTrieLoader.cpp index a88c0f5fe..3ac266bdb 100644 --- a/moses/Syntax/S2T/RuleTrieLoader.cpp +++ b/moses/Syntax/S2T/RuleTrieLoader.cpp @@ -125,9 +125,10 @@ bool RuleTrieLoader::Load(const std::vector<FactorType> &input, targetPhrase->GetScoreBreakdown().Assign(&ff, scoreVector); targetPhrase->EvaluateInIsolation(sourcePhrase, ff.GetFeaturesToApply()); - TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection( - trie, sourcePhrase, *targetPhrase, sourceLHS); - phraseColl.Add(targetPhrase); + TargetPhraseCollection::shared_ptr phraseColl + = GetOrCreateTargetPhraseCollection(trie, sourcePhrase, + *targetPhrase, sourceLHS); + phraseColl->Add(targetPhrase); // not implemented correctly in memory pt. just delete it for now delete sourceLHS; diff --git a/moses/Syntax/S2T/RuleTrieScope3.cpp b/moses/Syntax/S2T/RuleTrieScope3.cpp index 7318f09d6..ceaee9501 100644 --- a/moses/Syntax/S2T/RuleTrieScope3.cpp +++ b/moses/Syntax/S2T/RuleTrieScope3.cpp @@ -33,7 +33,7 @@ void RuleTrieScope3::Node::Prune(std::size_t tableLimit) // Prune TargetPhraseCollections at this node. for (LabelMap::iterator p = m_labelMap.begin(); p != m_labelMap.end(); ++p) { - p->second.Prune(true, tableLimit); + p->second->Prune(true, tableLimit); } } @@ -50,7 +50,7 @@ void RuleTrieScope3::Node::Sort(std::size_t tableLimit) // Sort TargetPhraseCollections at this node. for (LabelMap::iterator p = m_labelMap.begin(); p != m_labelMap.end(); ++p) { - p->second.Sort(true, tableLimit); + p->second->Sort(true, tableLimit); } } @@ -75,9 +75,10 @@ RuleTrieScope3::Node *RuleTrieScope3::Node::GetOrCreateNonTerminalChild( return m_gapNode; } -TargetPhraseCollection & -RuleTrieScope3::Node::GetOrCreateTargetPhraseCollection( - const TargetPhrase &target) +TargetPhraseCollection::shared_ptr +RuleTrieScope3:: +Node:: +GetOrCreateTargetPhraseCollection(const TargetPhrase &target) { const AlignmentInfo &alignmentInfo = target.GetAlignNonTerm(); const std::size_t rank = alignmentInfo.GetSize(); @@ -94,12 +95,16 @@ RuleTrieScope3::Node::GetOrCreateTargetPhraseCollection( const Word &targetNonTerm = target.GetWord(targetNonTermIndex); vec.push_back(InsertLabel(i++, targetNonTerm)); } - - return m_labelMap[vec]; + TargetPhraseCollection::shared_ptr& ret = m_labelMap[vec]; + if (!ret) ret.reset(new TargetPhraseCollection); + return ret; } -TargetPhraseCollection &RuleTrieScope3::GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS) +TargetPhraseCollection::shared_ptr +RuleTrieScope3:: +GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) { Node &currNode = GetOrCreateNode(source, target, sourceLHS); return currNode.GetOrCreateTargetPhraseCollection(target); diff --git a/moses/Syntax/S2T/RuleTrieScope3.h b/moses/Syntax/S2T/RuleTrieScope3.h index 5909b6509..97c939961 100644 --- a/moses/Syntax/S2T/RuleTrieScope3.h +++ b/moses/Syntax/S2T/RuleTrieScope3.h @@ -35,7 +35,7 @@ public: SymbolEqualityPred> TerminalMap; typedef boost::unordered_map<std::vector<int>, - TargetPhraseCollection> LabelMap; + TargetPhraseCollection::shared_ptr> LabelMap; ~Node() { delete m_gapNode; @@ -61,8 +61,8 @@ public: Node *GetOrCreateNonTerminalChild(const Word &targetNonTerm); - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const TargetPhrase &); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const TargetPhrase &); bool IsLeaf() const { return m_terminalMap.empty() && m_gapNode == NULL; @@ -106,8 +106,10 @@ public: bool HasPreterminalRule(const Word &) const; private: - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS); Node &GetOrCreateNode(const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); diff --git a/moses/Syntax/SHyperedgeBundle.h b/moses/Syntax/SHyperedgeBundle.h index 54eda73bc..d6e903529 100644 --- a/moses/Syntax/SHyperedgeBundle.h +++ b/moses/Syntax/SHyperedgeBundle.h @@ -17,7 +17,7 @@ struct PVertex; struct SHyperedgeBundle { float inputWeight; std::vector<const SVertexStack*> stacks; - const TargetPhraseCollection *translations; + TargetPhraseCollection::shared_ptr translations; friend void swap(SHyperedgeBundle &x, SHyperedgeBundle &y) { using std::swap; diff --git a/moses/Syntax/SVertex.cpp b/moses/Syntax/SVertex.cpp index 32650b2a8..cd4c1c666 100644 --- a/moses/Syntax/SVertex.cpp +++ b/moses/Syntax/SVertex.cpp @@ -1,7 +1,5 @@ #include "SVertex.h" - #include "moses/FF/FFState.h" - #include "SHyperedge.h" namespace Moses @@ -18,11 +16,38 @@ SVertex::~SVertex() delete *p; } // Delete FFState objects. - for (std::vector<FFState*>::iterator p = state.begin(); - p != state.end(); ++p) { + for (std::vector<FFState*>::iterator p = states.begin(); + p != states.end(); ++p) { delete *p; } } +size_t SVertex::hash() const +{ + size_t seed; + + // states + for (size_t i = 0; i < states.size(); ++i) { + const FFState *state = states[i]; + size_t hash = state->hash(); + boost::hash_combine(seed, hash); + } + return seed; + +} + +bool SVertex::operator==(const SVertex& other) const +{ + // states + for (size_t i = 0; i < states.size(); ++i) { + const FFState &thisState = *states[i]; + const FFState &otherState = *other.states[i]; + if (thisState != otherState) { + return false; + } + } + return true; +} + } // Syntax } // Moses diff --git a/moses/Syntax/SVertex.h b/moses/Syntax/SVertex.h index e596cb442..9a5392d30 100644 --- a/moses/Syntax/SVertex.h +++ b/moses/Syntax/SVertex.h @@ -1,6 +1,7 @@ #pragma once #include <vector> +#include <stddef.h> namespace Moses { @@ -23,7 +24,12 @@ struct SVertex { SHyperedge *best; std::vector<SHyperedge*> recombined; const PVertex *pvertex; - std::vector<FFState*> state; + std::vector<FFState*> states; + + // for unordered_set in stack + size_t hash() const; + bool operator==(const SVertex& other) const; + }; } // Syntax diff --git a/moses/Syntax/SVertexRecombinationOrderer.h b/moses/Syntax/SVertexRecombinationOrderer.h index fcabed04a..a91a3b125 100644 --- a/moses/Syntax/SVertexRecombinationOrderer.h +++ b/moses/Syntax/SVertexRecombinationOrderer.h @@ -9,26 +9,18 @@ namespace Moses namespace Syntax { -struct SVertexRecombinationOrderer { + +class SVertexRecombinationUnordered +{ public: - bool operator()(const SVertex &x, const SVertex &y) const { - int comp = 0; - for (std::size_t i = 0; i < x.state.size(); ++i) { - if (x.state[i] == NULL || y.state[i] == NULL) { - comp = x.state[i] - y.state[i]; - } else { - comp = x.state[i]->Compare(*y.state[i]); - } - if (comp != 0) { - return comp < 0; - } - } - return false; + size_t operator()(const SVertex* hypo) const { + return hypo->hash(); } - bool operator()(const SVertex *x, const SVertex *y) const { - return operator()(*x, *y); + bool operator()(const SVertex* hypoA, const SVertex* hypoB) const { + return (*hypoA) == (*hypoB); } + }; } // Syntax diff --git a/moses/Syntax/T2S/GlueRuleSynthesizer.cpp b/moses/Syntax/T2S/GlueRuleSynthesizer.cpp index 9c6dd91ab..f50f84629 100644 --- a/moses/Syntax/T2S/GlueRuleSynthesizer.cpp +++ b/moses/Syntax/T2S/GlueRuleSynthesizer.cpp @@ -17,9 +17,9 @@ void GlueRuleSynthesizer::SynthesizeRule(const InputTree::Node &node) const Word &sourceLhs = node.pvertex.symbol; boost::scoped_ptr<Phrase> sourceRhs(SynthesizeSourcePhrase(node)); TargetPhrase *tp = SynthesizeTargetPhrase(node, *sourceRhs); - TargetPhraseCollection &tpc = GetOrCreateTargetPhraseCollection( - m_ruleTrie, sourceLhs, *sourceRhs); - tpc.Add(tp); + TargetPhraseCollection::shared_ptr tpc + = GetOrCreateTargetPhraseCollection(m_ruleTrie, sourceLhs, *sourceRhs); + tpc->Add(tp); } Phrase *GlueRuleSynthesizer::SynthesizeSourcePhrase(const InputTree::Node &node) @@ -47,7 +47,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase( TargetPhrase *targetPhrase = new TargetPhrase(); - std::ostringstream alignmentSS; + util::StringStream alignmentSS; for (std::size_t i = 0; i < node.children.size(); ++i) { const Word &symbol = node.children[i]->pvertex.symbol; if (symbol.IsNonTerminal()) { diff --git a/moses/Syntax/T2S/HyperTree.h b/moses/Syntax/T2S/HyperTree.h index 800700365..66e7a7de6 100644 --- a/moses/Syntax/T2S/HyperTree.h +++ b/moses/Syntax/T2S/HyperTree.h @@ -48,11 +48,11 @@ public: const Node *GetChild(const HyperPath::NodeSeq &) const; - const TargetPhraseCollection &GetTargetPhraseCollection() const + const TargetPhraseCollection::shared_ptr GetTargetPhraseCollection() const return m_targetPhraseCollection; } - TargetPhraseCollection &GetTargetPhraseCollection() + TargetPhraseCollection::shared_ptr GetTargetPhraseCollection() return m_targetPhraseCollection; } @@ -76,7 +76,7 @@ const Node &GetRootNode() const private: friend class RuleTrieCreator; -TargetPhraseCollection &GetOrCreateTargetPhraseCollection( +TargetPhraseCollection::shared_ptr GetOrCreateTargetPhraseCollection( const Word &sourceLHS, const Phrase &sourceRHS); Node &GetOrCreateNode(const Phrase &sourceRHS); diff --git a/moses/Syntax/T2S/Manager-inl.h b/moses/Syntax/T2S/Manager-inl.h index 344d804e7..46d8b7177 100644 --- a/moses/Syntax/T2S/Manager-inl.h +++ b/moses/Syntax/T2S/Manager-inl.h @@ -245,7 +245,7 @@ void Manager<RuleMatcher>::RecombineAndSort( // head pointers are updated to point to the vertex instances in the map and // any 'duplicate' vertices are deleted. // TODO Set? - typedef std::map<SVertex *, SVertex *, SVertexRecombinationOrderer> Map; + typedef boost::unordered_map<SVertex *, SVertex *, SVertexRecombinationUnordered, SVertexRecombinationUnordered> Map; Map map; for (std::vector<SHyperedge*>::const_iterator p = buffer.begin(); p != buffer.end(); ++p) { diff --git a/moses/Syntax/T2S/RuleMatcherSCFG-inl.h b/moses/Syntax/T2S/RuleMatcherSCFG-inl.h index b782411a4..0eb7cbe2c 100644 --- a/moses/Syntax/T2S/RuleMatcherSCFG-inl.h +++ b/moses/Syntax/T2S/RuleMatcherSCFG-inl.h @@ -61,7 +61,7 @@ void RuleMatcherSCFG<Callback>::Match(const InputTree::Node &inNode, if (candidate.pvertex.span.GetEndPos() == inNode.pvertex.span.GetEndPos()) { // Check if the trie node has any rules with a LHS that match inNode. const Word &lhs = inNode.pvertex.symbol; - const TargetPhraseCollection *tpc = + TargetPhraseCollection::shared_ptr tpc = newTrieNode.GetTargetPhraseCollection(lhs); if (tpc) { m_hyperedge.label.translations = tpc; diff --git a/moses/Syntax/T2S/RuleTrie.cpp b/moses/Syntax/T2S/RuleTrie.cpp index 0fc7bf24c..fb5ae7108 100644 --- a/moses/Syntax/T2S/RuleTrie.cpp +++ b/moses/Syntax/T2S/RuleTrie.cpp @@ -35,7 +35,7 @@ void RuleTrie::Node::Prune(std::size_t tableLimit) // Prune TargetPhraseCollections at this node. for (TPCMap::iterator p = m_targetPhraseCollections.begin(); p != m_targetPhraseCollections.end(); ++p) { - p->second.Prune(true, tableLimit); + p->second->Prune(true, tableLimit); } } @@ -54,17 +54,21 @@ void RuleTrie::Node::Sort(std::size_t tableLimit) // Sort TargetPhraseCollections at this node. for (TPCMap::iterator p = m_targetPhraseCollections.begin(); p != m_targetPhraseCollections.end(); ++p) { - p->second.Sort(true, tableLimit); + p->second->Sort(true, tableLimit); } } -RuleTrie::Node *RuleTrie::Node::GetOrCreateChild( - const Word &sourceTerm) +RuleTrie::Node* +RuleTrie::Node:: +GetOrCreateChild(const Word &sourceTerm) { return &m_sourceTermMap[sourceTerm]; } -RuleTrie::Node *RuleTrie::Node::GetOrCreateNonTerminalChild(const Word &targetNonTerm) +RuleTrie::Node * +RuleTrie:: +Node:: +GetOrCreateNonTerminalChild(const Word &targetNonTerm) { UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(), "Not a non-terminal: " << targetNonTerm); @@ -72,42 +76,52 @@ RuleTrie::Node *RuleTrie::Node::GetOrCreateNonTerminalChild(const Word &targetNo return &m_nonTermMap[targetNonTerm]; } -TargetPhraseCollection &RuleTrie::Node::GetOrCreateTargetPhraseCollection( - const Word &sourceLHS) +TargetPhraseCollection::shared_ptr +RuleTrie:: +Node:: +GetOrCreateTargetPhraseCollection(const Word &sourceLHS) { UTIL_THROW_IF2(!sourceLHS.IsNonTerminal(), "Not a non-terminal: " << sourceLHS); - return m_targetPhraseCollections[sourceLHS]; + TargetPhraseCollection::shared_ptr& foo + = m_targetPhraseCollections[sourceLHS]; + if (!foo) foo.reset(new TargetPhraseCollection); + return foo; } -const RuleTrie::Node *RuleTrie::Node::GetChild( - const Word &sourceTerm) const +RuleTrie::Node const* +RuleTrie:: +Node:: +GetChild(const Word &sourceTerm) const { - UTIL_THROW_IF2(sourceTerm.IsNonTerminal(), - "Not a terminal: " << sourceTerm); - + UTIL_THROW_IF2(sourceTerm.IsNonTerminal(), "Not a terminal: " << sourceTerm); SymbolMap::const_iterator p = m_sourceTermMap.find(sourceTerm); return (p == m_sourceTermMap.end()) ? NULL : &p->second; } -const RuleTrie::Node *RuleTrie::Node::GetNonTerminalChild( - const Word &targetNonTerm) const +RuleTrie::Node const* +RuleTrie:: +Node:: +GetNonTerminalChild(const Word &targetNonTerm) const { UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(), "Not a non-terminal: " << targetNonTerm); - SymbolMap::const_iterator p = m_nonTermMap.find(targetNonTerm); return (p == m_nonTermMap.end()) ? NULL : &p->second; } -TargetPhraseCollection &RuleTrie::GetOrCreateTargetPhraseCollection( - const Word &sourceLHS, const Phrase &sourceRHS) +TargetPhraseCollection::shared_ptr +RuleTrie:: +GetOrCreateTargetPhraseCollection +( const Word &sourceLHS, const Phrase &sourceRHS ) { Node &currNode = GetOrCreateNode(sourceRHS); return currNode.GetOrCreateTargetPhraseCollection(sourceLHS); } -RuleTrie::Node &RuleTrie::GetOrCreateNode(const Phrase &sourceRHS) +RuleTrie::Node & +RuleTrie:: +GetOrCreateNode(const Phrase &sourceRHS) { const std::size_t size = sourceRHS.GetSize(); diff --git a/moses/Syntax/T2S/RuleTrie.h b/moses/Syntax/T2S/RuleTrie.h index 2807f6e0e..16b9e735f 100644 --- a/moses/Syntax/T2S/RuleTrie.h +++ b/moses/Syntax/T2S/RuleTrie.h @@ -32,7 +32,7 @@ public: typedef boost::unordered_map<Word, Node, SymbolHasher, SymbolEqualityPred> SymbolMap; - typedef boost::unordered_map<Word, TargetPhraseCollection, + typedef boost::unordered_map<Word, TargetPhraseCollection::shared_ptr, SymbolHasher, SymbolEqualityPred> TPCMap; bool IsLeaf() const { @@ -48,15 +48,18 @@ public: Node *GetOrCreateChild(const Word &sourceTerm); Node *GetOrCreateNonTerminalChild(const Word &targetNonTerm); - TargetPhraseCollection &GetOrCreateTargetPhraseCollection(const Word &); + TargetPhraseCollection::shared_ptr GetOrCreateTargetPhraseCollection(const Word &); const Node *GetChild(const Word &sourceTerm) const; const Node *GetNonTerminalChild(const Word &targetNonTerm) const; - const TargetPhraseCollection *GetTargetPhraseCollection( - const Word &sourceLHS) const { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection(const Word &sourceLHS) const { TPCMap::const_iterator p = m_targetPhraseCollections.find(sourceLHS); - return p == m_targetPhraseCollections.end() ? 0 : &(p->second); + if (p != m_targetPhraseCollections.end()) + return p->second; + else + return TargetPhraseCollection::shared_ptr(); } // FIXME IS there any reason to distinguish these two for T2S? @@ -83,8 +86,9 @@ public: private: friend class RuleTrieCreator; - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Word &sourceLHS, const Phrase &sourceRHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection + (const Word &sourceLHS, const Phrase &sourceRHS); Node &GetOrCreateNode(const Phrase &sourceRHS); diff --git a/moses/Syntax/T2S/RuleTrieCreator.h b/moses/Syntax/T2S/RuleTrieCreator.h index fd29d3838..af5e976b5 100644 --- a/moses/Syntax/T2S/RuleTrieCreator.h +++ b/moses/Syntax/T2S/RuleTrieCreator.h @@ -21,7 +21,7 @@ protected: // Provide access to RuleTrie's private // GetOrCreateTargetPhraseCollection function. - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( + TargetPhraseCollection::shared_ptr GetOrCreateTargetPhraseCollection( RuleTrie &trie, const Word &sourceLHS, const Phrase &sourceRHS) { return trie.GetOrCreateTargetPhraseCollection(sourceLHS, sourceRHS); } diff --git a/moses/Syntax/T2S/RuleTrieLoader.cpp b/moses/Syntax/T2S/RuleTrieLoader.cpp index 81924f05d..f0646b1f7 100644 --- a/moses/Syntax/T2S/RuleTrieLoader.cpp +++ b/moses/Syntax/T2S/RuleTrieLoader.cpp @@ -55,7 +55,9 @@ bool RuleTrieLoader::Load(const std::vector<FactorType> &input, std::vector<float> scoreVector; StringPiece line; - double_conversion::StringToDoubleConverter converter(double_conversion::StringToDoubleConverter::NO_FLAGS, NAN, NAN, "inf", "nan"); + int noflags = double_conversion::StringToDoubleConverter::NO_FLAGS; + double_conversion::StringToDoubleConverter + converter(noflags, NAN, NAN, "inf", "nan"); while(true) { try { @@ -132,9 +134,9 @@ bool RuleTrieLoader::Load(const std::vector<FactorType> &input, targetPhrase->GetScoreBreakdown().Assign(&ff, scoreVector); targetPhrase->EvaluateInIsolation(sourcePhrase, ff.GetFeaturesToApply()); - TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection( - trie, *sourceLHS, sourcePhrase); - phraseColl.Add(targetPhrase); + TargetPhraseCollection::shared_ptr phraseColl + = GetOrCreateTargetPhraseCollection(trie, *sourceLHS, sourcePhrase); + phraseColl->Add(targetPhrase); // not implemented correctly in memory pt. just delete it for now delete sourceLHS; diff --git a/moses/TargetPhrase.h b/moses/TargetPhrase.h index 56ed27af3..252afefc1 100644 --- a/moses/TargetPhrase.h +++ b/moses/TargetPhrase.h @@ -230,29 +230,6 @@ void swap(TargetPhrase &first, TargetPhrase &second); std::ostream& operator<<(std::ostream&, const TargetPhrase&); -/** - * Hasher that looks at source and target phrase. - **/ -struct TargetPhraseHasher { - inline size_t operator()(const TargetPhrase& targetPhrase) const { - size_t seed = 0; - boost::hash_combine(seed, targetPhrase); - boost::hash_combine(seed, targetPhrase.GetAlignTerm()); - boost::hash_combine(seed, targetPhrase.GetAlignNonTerm()); - - return seed; - } -}; - -struct TargetPhraseComparator { - inline bool operator()(const TargetPhrase& lhs, const TargetPhrase& rhs) const { - return lhs.Compare(rhs) == 0 && - lhs.GetAlignTerm() == rhs.GetAlignTerm() && - lhs.GetAlignNonTerm() == rhs.GetAlignNonTerm(); - } - -}; - } #endif diff --git a/moses/TargetPhraseCollection.h b/moses/TargetPhraseCollection.h index d61ff2c4f..47b3afd57 100644 --- a/moses/TargetPhraseCollection.h +++ b/moses/TargetPhraseCollection.h @@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <iostream> #include "TargetPhrase.h" #include "Util.h" +#include <boost/shared_ptr.hpp> namespace Moses { @@ -43,6 +44,8 @@ public: // iters typedef CollType::iterator iterator; typedef CollType::const_iterator const_iterator; + typedef boost::shared_ptr<TargetPhraseCollection> shared_ptr; + typedef boost::shared_ptr<TargetPhraseCollection const> shared_const_ptr; TargetPhrase const* operator[](size_t const i) const { @@ -127,6 +130,9 @@ protected: std::vector<Phrase> m_sourcePhrases; public: + typedef boost::shared_ptr<TargetPhraseCollectionWithSourcePhrase> shared_ptr; + typedef boost::shared_ptr<TargetPhraseCollectionWithSourcePhrase const> shared_const_ptr; + const std::vector<Phrase> &GetSourcePhrases() const { return m_sourcePhrases; } diff --git a/moses/TranslationAnalysis.cpp b/moses/TranslationAnalysis.cpp index ed948f6b6..ebca957ce 100644 --- a/moses/TranslationAnalysis.cpp +++ b/moses/TranslationAnalysis.cpp @@ -10,6 +10,7 @@ #include "moses/FF/StatefulFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h" #include "moses/LM/Base.h" +#include "util/string_stream.hh" using namespace Moses; @@ -40,8 +41,9 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo) if (doLMStats) lmAcc.resize((*tpi)->GetLMStats()->size(), 0); for (; tpi != translationPath.end(); ++tpi) { - std::ostringstream sms; - std::ostringstream tms; + util::StringStream sms; + + util::StringStream tms; std::string target = (*tpi)->GetTargetPhraseStringRep(); std::string source = (*tpi)->GetSourcePhraseStringRep(); WordsRange twr = (*tpi)->GetCurrTargetWordsRange(); diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp index 54f172d1e..61ec03892 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp @@ -167,10 +167,10 @@ void ChartRuleLookupManagerMemory::AddAndExtend( size_t endPos) { - const TargetPhraseCollection &tpc = node->GetTargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpc = node->GetTargetPhraseCollection(); // add target phrase collection (except if rule is empty or a unary non-terminal rule) - if (!tpc.IsEmpty() && (m_stackVec.empty() || endPos != m_unaryPos)) { - m_completedRules[endPos].Add(tpc, m_stackVec, m_stackScores, *m_outColl); + if (!tpc->IsEmpty() && (m_stackVec.empty() || endPos != m_unaryPos)) { + m_completedRules[endPos].Add(*tpc, m_stackVec, m_stackScores, *m_outColl); } // get all further extensions of rule (until reaching end of sentence or max-chart-span) diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp index e090ee1ae..26ecb80e2 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp @@ -167,10 +167,11 @@ void ChartRuleLookupManagerMemoryPerSentence::AddAndExtend( size_t endPos) { - const TargetPhraseCollection &tpc = node->GetTargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpc + = node->GetTargetPhraseCollection(); // add target phrase collection (except if rule is empty or a unary non-terminal rule) - if (!tpc.IsEmpty() && (m_stackVec.empty() || endPos != m_unaryPos)) { - m_completedRules[endPos].Add(tpc, m_stackVec, m_stackScores, *m_outColl); + if (!tpc->IsEmpty() && (m_stackVec.empty() || endPos != m_unaryPos)) { + m_completedRules[endPos].Add(*tpc, m_stackVec, m_stackScores, *m_outColl); } // get all further extensions of rule (until reaching end of sentence or max-chart-span) diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp index 349fc4cbc..102fa82cf 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp @@ -64,11 +64,12 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( ChartRuleLookupManagerOnDisk::~ChartRuleLookupManagerOnDisk() { - std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache; - for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache) { - delete iterCache->second; - } - m_cache.clear(); + // not needed any more due to the switch to shared pointers + // std::map<uint64_t, TargetPhraseCollection::shared_ptr >::const_iterator iterCache; + // for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache) { + // iterCache->second.reset(); + // } + // m_cache.clear(); RemoveAllInColl(m_expandableDottedRuleListVec); RemoveAllInColl(m_sourcePhraseNode); @@ -236,14 +237,16 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( if (sourceLHSBerkeleyDb == NULL) continue; - const TargetPhraseCollection *targetPhraseCollection = NULL; - const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper); + TargetPhraseCollection::shared_ptr targetPhraseCollection; + const OnDiskPt::PhraseNode *node + = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper); if (node) { uint64_t tpCollFilePos = node->GetValue(); - std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos); + std::map<uint64_t, TargetPhraseCollection::shared_ptr >::const_iterator iterCache = m_cache.find(tpCollFilePos); if (iterCache == m_cache.end()) { - const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); + OnDiskPt::TargetPhraseCollection::shared_ptr tpcollBerkeleyDb + = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); std::vector<float> weightT = staticData.GetWeights(&m_dictionary); targetPhraseCollection @@ -254,7 +257,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( ,m_dbWrapper.GetVocab() ,true); - delete tpcollBerkeleyDb; + tpcollBerkeleyDb.reset(); m_cache[tpCollFilePos] = targetPhraseCollection; } else { // just get out of cache diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h index 42cbdcc46..dee9cc202 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h @@ -55,7 +55,7 @@ private: const std::vector<FactorType> &m_inputFactorsVec; const std::vector<FactorType> &m_outputFactorsVec; std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec; - std::map<uint64_t, const TargetPhraseCollection*> m_cache; + std::map<uint64_t, TargetPhraseCollection::shared_ptr > m_cache; std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode; }; diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp index 53011e5ac..7db3e14cd 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp @@ -48,7 +48,7 @@ ChartRuleLookupManagerSkeleton::ChartRuleLookupManagerSkeleton( ChartRuleLookupManagerSkeleton::~ChartRuleLookupManagerSkeleton() { - RemoveAllInColl(m_tpColl); + // RemoveAllInColl(m_tpColl); } void ChartRuleLookupManagerSkeleton::GetChartRuleCollection( @@ -58,7 +58,7 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection( { //m_tpColl.push_back(TargetPhraseCollection()); //TargetPhraseCollection &tpColl = m_tpColl.back(); - TargetPhraseCollection *tpColl = new TargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpColl(new TargetPhraseCollection); m_tpColl.push_back(tpColl); const WordsRange &range = inputPath.GetWordsRange(); @@ -73,7 +73,9 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection( outColl.Add(*tpColl, m_stackVec, range); } -TargetPhrase *ChartRuleLookupManagerSkeleton::CreateTargetPhrase(const Word &sourceWord) const +TargetPhrase * +ChartRuleLookupManagerSkeleton:: +CreateTargetPhrase(const Word &sourceWord) const { // create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:' string str = sourceWord.GetFactor(0)->GetString().as_string(); diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h index 0c141d2ef..219d7b2b6 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h @@ -49,7 +49,7 @@ private: TargetPhrase *CreateTargetPhrase(const Word &sourceWord) const; StackVec m_stackVec; - std::vector<TargetPhraseCollection*> m_tpColl; + std::vector<TargetPhraseCollection::shared_ptr > m_tpColl; const SkeletonPT &m_skeletonPT; }; diff --git a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h index 84b583df6..95823328f 100644 --- a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h +++ b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h @@ -119,4 +119,4 @@ private: } // namespace Moses -#endif
\ No newline at end of file +#endif diff --git a/moses/TranslationModel/CompactPT/BlockHashIndex.cpp b/moses/TranslationModel/CompactPT/BlockHashIndex.cpp index 27209f5bc..4130bd6a4 100644 --- a/moses/TranslationModel/CompactPT/BlockHashIndex.cpp +++ b/moses/TranslationModel/CompactPT/BlockHashIndex.cpp @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "BlockHashIndex.h" #include "CmphStringVectorAdapter.h" #include "util/exception.hh" +#include "util/string_stream.hh" #ifdef HAVE_CMPH #include "cmph.h" @@ -98,11 +99,11 @@ size_t BlockHashIndex::GetFprint(const char* key) const size_t BlockHashIndex::GetHash(size_t i, const char* key) { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_mutex); -#endif - if(m_hashes[i] == 0) - LoadRange(i); +//#ifdef WITH_THREADS +// boost::mutex::scoped_lock lock(m_mutex); +//#endif + //if(m_hashes[i] == 0) + //LoadRange(i); #ifdef HAVE_CMPH size_t idx = cmph_search((cmph_t*)m_hashes[i], key, (cmph_uint32) strlen(key)); #else @@ -322,9 +323,10 @@ size_t BlockHashIndex::GetSize() const void BlockHashIndex::KeepNLastRanges(float ratio, float tolerance) { -#ifdef WITH_THREADS + /* + #ifdef WITH_THREADS boost::mutex::scoped_lock lock(m_mutex); -#endif + #endif size_t n = m_hashes.size() * ratio; size_t max = n * (1 + tolerance); if(m_numLoadedRanges > max) { @@ -338,7 +340,7 @@ void BlockHashIndex::KeepNLastRanges(float ratio, float tolerance) for(LastLoaded::reverse_iterator it = lastLoaded.rbegin() + size_t(n * (1 - tolerance)); it != lastLoaded.rend(); it++) DropRange(it->second); - } + }*/ } void BlockHashIndex::CalcHash(size_t current, void* source_void) @@ -366,10 +368,10 @@ void BlockHashIndex::CalcHash(size_t current, void* source_void) if(lastKey > temp) { if(source->nkeys != 2 || temp != "###DUMMY_KEY###") { - std::stringstream strme; - strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl; - strme << "1: " << lastKey << std::endl; - strme << "2: " << temp << std::endl; + util::StringStream strme; + strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n"; + strme << "1: " << lastKey << "\n"; + strme << "2: " << temp << "\n"; UTIL_THROW2(strme.str()); } } diff --git a/moses/TranslationModel/CompactPT/BlockHashIndex.h b/moses/TranslationModel/CompactPT/BlockHashIndex.h index 0f20fa1b2..3de46272a 100644 --- a/moses/TranslationModel/CompactPT/BlockHashIndex.h +++ b/moses/TranslationModel/CompactPT/BlockHashIndex.h @@ -34,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "StringVector.h" #include "PackedArray.h" #include "util/exception.hh" +#include "util/string_stream.hh" #ifdef WITH_THREADS #include "moses/ThreadPool.h" @@ -145,10 +146,10 @@ public: size_t current = m_landmarks.size(); if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) { - std::stringstream strme; - strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl; - strme << "1: " << m_landmarks.back().str() << std::endl; - strme << "2: " << keys[0] << std::endl; + util::StringStream strme; + strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n"; + strme << "1: " << m_landmarks.back().str() << "\n"; + strme << "2: " << keys[0] << "\n"; UTIL_THROW2(strme.str()); } diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp index cd71b1776..729df525b 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp @@ -155,10 +155,12 @@ LexicalReorderingTableCompact:: Load(std::string filePath) { std::FILE* pFile = std::fopen(filePath.c_str(), "r"); - if(m_inMemory) - m_hash.Load(pFile); - else - m_hash.LoadIndex(pFile); + UTIL_THROW_IF2(pFile == NULL, "File " << filePath << " could not be opened"); + + //if(m_inMemory) + m_hash.Load(pFile); + //else + //m_hash.LoadIndex(pFile); size_t read = 0; read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile); diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp index afed99057..880067ba4 100644 --- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp +++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp @@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <algorithm> #include <sys/stat.h> #include <boost/algorithm/string/predicate.hpp> +#include <boost/thread/tss.hpp> #include "PhraseDictionaryCompact.h" #include "moses/FactorCollection.h" @@ -43,6 +44,8 @@ using namespace boost::algorithm; namespace Moses { +typename PhraseDictionaryCompact::SentenceCache PhraseDictionaryCompact::m_sentenceCache; + PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line) :PhraseDictionary(line, true) ,m_inMemory(true) @@ -75,12 +78,12 @@ void PhraseDictionaryCompact::Load() std::FILE* pFile = std::fopen(tFilePath.c_str() , "r"); size_t indexSize; - if(m_inMemory) - // Load source phrase index into memory - indexSize = m_hash.Load(pFile); - else - // Keep source phrase index on disk - indexSize = m_hash.LoadIndex(pFile); + //if(m_inMemory) + // Load source phrase index into memory + indexSize = m_hash.Load(pFile); +// else + // Keep source phrase index on disk + //indexSize = m_hash.LoadIndex(pFile); size_t coderSize = m_phraseDecoder->Load(pFile); @@ -104,14 +107,15 @@ void PhraseDictionaryCompact::Load() // } // }; -const TargetPhraseCollection* +TargetPhraseCollection::shared_ptr PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const { + TargetPhraseCollection::shared_ptr ret; // There is no souch source phrase if source phrase is longer than longest // observed source phrase during compilation if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength()) - return NULL; + return ret; // Retrieve target phrase collection from phrase table TargetPhraseVectorPtr decodedPhraseColl @@ -119,7 +123,7 @@ PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &s if(decodedPhraseColl != NULL && decodedPhraseColl->size()) { TargetPhraseVectorPtr tpv(new TargetPhraseVector(*decodedPhraseColl)); - TargetPhraseCollection* phraseColl = new TargetPhraseCollection(); + TargetPhraseCollection::shared_ptr phraseColl(new TargetPhraseCollection); // Score phrases and if possible apply ttable_limit TargetPhraseVector::iterator nth = @@ -136,7 +140,7 @@ PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &s return phraseColl; } else - return NULL; + return ret; } TargetPhraseVectorPtr @@ -160,15 +164,11 @@ PhraseDictionaryCompact::~PhraseDictionaryCompact() //TO_STRING_BODY(PhraseDictionaryCompact) -void PhraseDictionaryCompact::CacheForCleanup(TargetPhraseCollection* tpc) +void PhraseDictionaryCompact::CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_sentenceMutex); - PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()]; -#else - PhraseCache &ref = m_sentenceCache; -#endif - ref.push_back(tpc); + if(!m_sentenceCache.get()) + m_sentenceCache.reset(new PhraseCache()); + m_sentenceCache->push_back(tpc); } void PhraseDictionaryCompact::AddEquivPhrase(const Phrase &source, @@ -176,23 +176,17 @@ void PhraseDictionaryCompact::AddEquivPhrase(const Phrase &source, void PhraseDictionaryCompact::CleanUpAfterSentenceProcessing(const InputType &source) { - if(!m_inMemory) - m_hash.KeepNLastRanges(0.01, 0.2); + if(!m_sentenceCache.get()) + m_sentenceCache.reset(new PhraseCache()); m_phraseDecoder->PruneCache(); + // for(PhraseCache::iterator it = m_sentenceCache->begin(); + // it != m_sentenceCache->end(); it++) + // it->reset(); -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_sentenceMutex); - PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()]; -#else - PhraseCache &ref = m_sentenceCache; -#endif - - for(PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) - delete *it; - - PhraseCache temp; - temp.swap(ref); + // PhraseCache temp; + // temp.swap(*m_sentenceCache); + m_sentenceCache->clear(); ReduceCache(); } diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h index 379bbe844..f1c3dd1d8 100644 --- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h +++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h @@ -51,14 +51,9 @@ protected: bool m_inMemory; bool m_useAlignmentInfo; - typedef std::vector<TargetPhraseCollection*> PhraseCache; -#ifdef WITH_THREADS - boost::mutex m_sentenceMutex; - typedef std::map<boost::thread::id, PhraseCache> SentenceCache; -#else - typedef PhraseCache SentenceCache; -#endif - SentenceCache m_sentenceCache; + typedef std::vector<TargetPhraseCollection::shared_ptr > PhraseCache; + typedef boost::thread_specific_ptr<PhraseCache> SentenceCache; + static SentenceCache m_sentenceCache; BlockHashIndex m_hash; PhraseDecoder* m_phraseDecoder; @@ -74,12 +69,12 @@ public: void Load(); - const TargetPhraseCollection* GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const; + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const; TargetPhraseVectorPtr GetTargetPhraseCollectionRaw(const Phrase &source) const; void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase); - void CacheForCleanup(TargetPhraseCollection* tpc); + void CacheForCleanup(TargetPhraseCollection::shared_ptr tpc); void CleanUpAfterSentenceProcessing(const InputType &source); virtual ChartRuleLookupManager *CreateRuleLookupManager( diff --git a/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp b/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp new file mode 100644 index 000000000..964ab4528 --- /dev/null +++ b/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp @@ -0,0 +1,32 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include "TargetPhraseCollectionCache.h" + +namespace Moses +{ + + +boost::thread_specific_ptr<typename TargetPhraseCollectionCache::CacheMap> +TargetPhraseCollectionCache::m_phraseCache; + +} + diff --git a/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h b/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h index 1d5ed0da3..e017a3c19 100644 --- a/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h +++ b/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h @@ -26,12 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <set> #include <vector> -#ifdef WITH_THREADS -#ifdef BOOST_HAS_PTHREADS -#include <boost/thread/mutex.hpp> -#endif -#endif - +#include <boost/thread/tss.hpp> #include <boost/shared_ptr.hpp> #include "moses/Phrase.h" @@ -63,12 +58,7 @@ private: }; typedef std::map<Phrase, LastUsed> CacheMap; - - CacheMap m_phraseCache; - -#ifdef WITH_THREADS - boost::mutex m_mutex; -#endif + static boost::thread_specific_ptr<CacheMap> m_phraseCache; public: @@ -80,31 +70,37 @@ public: } iterator Begin() { - return m_phraseCache.begin(); + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->begin(); } const_iterator Begin() const { - return m_phraseCache.begin(); + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->begin(); } iterator End() { - return m_phraseCache.end(); + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->end(); } const_iterator End() const { - return m_phraseCache.end(); + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->end(); } /** retrieve translations for source phrase from persistent cache **/ void Cache(const Phrase &sourcePhrase, TargetPhraseVectorPtr tpv, size_t bitsLeft = 0, size_t maxRank = 0) { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_mutex); -#endif - + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); // check if source phrase is already in cache - iterator it = m_phraseCache.find(sourcePhrase); - if(it != m_phraseCache.end()) + iterator it = m_phraseCache->find(sourcePhrase); + if(it != m_phraseCache->end()) // if found, just update clock it->second.m_clock = clock(); else { @@ -113,19 +109,17 @@ public: TargetPhraseVectorPtr tpv_temp(new TargetPhraseVector()); tpv_temp->resize(maxRank); std::copy(tpv->begin(), tpv->begin() + maxRank, tpv_temp->begin()); - m_phraseCache[sourcePhrase] = LastUsed(clock(), tpv_temp, bitsLeft); + (*m_phraseCache)[sourcePhrase] = LastUsed(clock(), tpv_temp, bitsLeft); } else - m_phraseCache[sourcePhrase] = LastUsed(clock(), tpv, bitsLeft); + (*m_phraseCache)[sourcePhrase] = LastUsed(clock(), tpv, bitsLeft); } } std::pair<TargetPhraseVectorPtr, size_t> Retrieve(const Phrase &sourcePhrase) { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_mutex); -#endif - - iterator it = m_phraseCache.find(sourcePhrase); - if(it != m_phraseCache.end()) { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + iterator it = m_phraseCache->find(sourcePhrase); + if(it != m_phraseCache->end()) { LastUsed &lu = it->second; lu.m_clock = clock(); return std::make_pair(lu.m_tpv, lu.m_bitsLeft); @@ -135,34 +129,31 @@ public: // if cache full, reduce void Prune() { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_mutex); -#endif - - if(m_phraseCache.size() > m_max * (1 + m_tolerance)) { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + if(m_phraseCache->size() > m_max * (1 + m_tolerance)) { typedef std::set<std::pair<clock_t, Phrase> > Cands; Cands cands; - for(CacheMap::iterator it = m_phraseCache.begin(); - it != m_phraseCache.end(); it++) { + for(CacheMap::iterator it = m_phraseCache->begin(); + it != m_phraseCache->end(); it++) { LastUsed &lu = it->second; cands.insert(std::make_pair(lu.m_clock, it->first)); } for(Cands::iterator it = cands.begin(); it != cands.end(); it++) { const Phrase& p = it->second; - m_phraseCache.erase(p); + m_phraseCache->erase(p); - if(m_phraseCache.size() < (m_max * (1 - m_tolerance))) + if(m_phraseCache->size() < (m_max * (1 - m_tolerance))) break; } } } void CleanUp() { -#ifdef WITH_THREADS - boost::mutex::scoped_lock lock(m_mutex); -#endif - m_phraseCache.clear(); + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + m_phraseCache->clear(); } }; diff --git a/moses/TranslationModel/PhraseDictionary.cpp b/moses/TranslationModel/PhraseDictionary.cpp index 643633be4..f27ae004e 100644 --- a/moses/TranslationModel/PhraseDictionary.cpp +++ b/moses/TranslationModel/PhraseDictionary.cpp @@ -35,14 +35,15 @@ namespace Moses { std::vector<PhraseDictionary*> PhraseDictionary::s_staticColl; -CacheColl::~CacheColl() -{ - for (iterator iter = begin(); iter != end(); ++iter) { - std::pair<const TargetPhraseCollection*, clock_t> &key = iter->second; - const TargetPhraseCollection *tps = key.first; - delete tps; - } -} +// CacheColl::~CacheColl() +// { +// // not needed any more since the switch to shared pointers +// // for (iterator iter = begin(); iter != end(); ++iter) { +// // std::pair<TargetPhraseCollection::shared_ptr , clock_t> &key = iter->second; +// // TargetPhraseCollection::shared_ptr tps = key.first; +// // delete tps; +// // } +// } PhraseDictionary::PhraseDictionary(const std::string &line, bool registerNow) : DecodeFeature(line, registerNow) @@ -60,9 +61,12 @@ ProvidesPrefixCheck() const return false; } -const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionLEGACY(const Phrase& src) const +TargetPhraseCollection::shared_ptr +PhraseDictionary:: +GetTargetPhraseCollectionLEGACY(const Phrase& src) const { - const TargetPhraseCollection *ret; + TargetPhraseCollection::shared_ptr ret; + typedef std::pair<TargetPhraseCollection::shared_ptr , clock_t> entry; if (m_maxCacheSize) { CacheColl &cache = GetCache(); @@ -74,18 +78,14 @@ const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionLEGACY( if (iter == cache.end()) { // not in cache, need to look up from phrase table ret = GetTargetPhraseCollectionNonCacheLEGACY(src); - if (ret) { - ret = new TargetPhraseCollection(*ret); + if (ret) { // make a copy + ret.reset(new TargetPhraseCollection(*ret)); } - - std::pair<const TargetPhraseCollection*, clock_t> value(ret, clock()); - cache[hash] = value; + cache[hash] = entry(ret, clock()); } else { // in cache. just use it - std::pair<const TargetPhraseCollection*, clock_t> &value = iter->second; - value.second = clock(); - - ret = value.first; + iter->second.second = clock(); + ret = iter->second.first; } } else { // don't use cache. look up from phrase table @@ -95,7 +95,7 @@ const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionLEGACY( return ret; } -TargetPhraseCollection const * +TargetPhraseCollection::shared_ptr PhraseDictionary:: GetTargetPhraseCollectionNonCacheLEGACY(const Phrase& src) const { @@ -103,7 +103,7 @@ GetTargetPhraseCollectionNonCacheLEGACY(const Phrase& src) const } -TargetPhraseCollectionWithSourcePhrase const* +TargetPhraseCollectionWithSourcePhrase::shared_ptr PhraseDictionary:: GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const& range) const { @@ -140,14 +140,14 @@ SetFeaturesToApply() } -// tell the Phrase Dictionary that the TargetPhraseCollection is not needed any more -void -PhraseDictionary:: -Release(TargetPhraseCollection const* tpc) const -{ - // do nothing by default - return; -} +// // tell the Phrase Dictionary that the TargetPhraseCollection is not needed any more +// void +// PhraseDictionary:: +// Release(ttasksptr const& ttask, TargetPhraseCollection const*& tpc) const +// { +// // do nothing by default +// return; +// } bool PhraseDictionary:: @@ -170,7 +170,7 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const } const Phrase &phrase = inputPath.GetPhrase(); - const TargetPhraseCollection *targetPhrases = this->GetTargetPhraseCollectionLEGACY(phrase); + TargetPhraseCollection::shared_ptr targetPhrases = this->GetTargetPhraseCollectionLEGACY(phrase); inputPath.SetTargetPhrases(*this, targetPhrases, NULL); } } @@ -180,7 +180,7 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const //void PhraseDictionary::SaveCache() const //{ // CacheColl &cache = GetCache(); -// for( std::map<size_t, std::pair<const TargetPhraseCollection*,clock_t> >::iterator iter, +// for( std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iter, // iter != cache.end(), // iter++ ) { // @@ -191,10 +191,10 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const //void PhraseDictionary::LoadCache() const //{ // CacheColl &cache = GetCache(); -// std::map<size_t, std::pair<const TargetPhraseCollection*,clock_t> >::iterator iter; +// std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iter; // iter = cache.begin(); // while( iter != cache.end() ) { -// std::map<size_t, std::pair<const TargetPhraseCollection*,clock_t> >::iterator iterRemove = iter++; +// std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iterRemove = iter++; // delete iterRemove->second.first; // cache.erase(iterRemove); // } @@ -225,11 +225,12 @@ void PhraseDictionary::ReduceCache() const while( iter != cache.end() ) { if (iter->second.second < cutoffLastUsedTime) { CacheColl::iterator iterRemove = iter++; - delete iterRemove->second.first; + // delete iterRemove->second.first; cache.erase(iterRemove); } else iter++; } - VERBOSE(2,"Reduced persistent translation option cache in " << reduceCacheTime << " seconds." << std::endl); + VERBOSE(2,"Reduced persistent translation option cache in " + << reduceCacheTime << " seconds." << std::endl); } CacheColl &PhraseDictionary::GetCache() const @@ -265,8 +266,8 @@ bool PhraseDictionary::SatisfyBackoff(const InputPath &inputPath) const // lookup translation only if no other translations InputPath::TargetPhrases::const_iterator iter; for (iter = inputPath.GetTargetPhrases().begin(); iter != inputPath.GetTargetPhrases().end(); ++iter) { - const std::pair<const TargetPhraseCollection*, const void*> &temp = iter->second; - const TargetPhraseCollection *tpCollPrev = temp.first; + const std::pair<TargetPhraseCollection::shared_ptr , const void*> &temp = iter->second; + TargetPhraseCollection::shared_ptr tpCollPrev = temp.first; if (tpCollPrev && tpCollPrev->GetSize()) { // already have translation from another pt. Don't create translations diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h index 9cdb81089..5798abb37 100644 --- a/moses/TranslationModel/PhraseDictionary.h +++ b/moses/TranslationModel/PhraseDictionary.h @@ -55,15 +55,18 @@ class ChartCellCollectionBase; class ChartRuleLookupManager; class ChartParser; -class CacheColl : public boost::unordered_map<size_t, std::pair<const TargetPhraseCollection*, clock_t> > -{ -// 1st = hash of source phrase/ address of phrase-table node -// 2nd = all translations -// 3rd = time of last access - -public: - ~CacheColl(); -}; +// typedef std::pair<TargetPhraseCollection::shared_ptr, clock_t> TPCollLastUse; +typedef std::pair<TargetPhraseCollection::shared_ptr, clock_t> CacheCollEntry; +typedef boost::unordered_map<size_t, CacheCollEntry> CacheColl; +// class CacheColl : public boost::unordered_map<size_t, TPCollLastUse> +// { +// // 1st = hash of source phrase/ address of phrase-table node +// // 2nd = all translations +// // 3rd = time of last access + +// public: +// ~CacheColl(); +// }; /** * Abstract base class for phrase dictionaries (tables). @@ -95,9 +98,9 @@ public: return m_id; } - virtual - void - Release(TargetPhraseCollection const* tpc) const; + // virtual + // void + // Release(ttasksptr const& ttask, TargetPhraseCollection const*& tpc) const; /// return true if phrase table entries starting with /phrase/ // exist in the table. @@ -111,24 +114,21 @@ public: //! find list of translations that can translates src. Only for phrase input public: - virtual - TargetPhraseCollection const * + virtual TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionLEGACY(const Phrase& src) const; - virtual - TargetPhraseCollection const * - GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) const { + virtual TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, + Phrase const& src) const { return GetTargetPhraseCollectionLEGACY(src); } - virtual - void + virtual void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; - virtual - void - GetTargetPhraseCollectionBatch(ttasksptr const& ttask, - const InputPathList &inputPathQueue) const { + virtual void + GetTargetPhraseCollectionBatch + (ttasksptr const& ttask, InputPathList const& inputPathQueue) const { GetTargetPhraseCollectionBatch(inputPathQueue); } @@ -157,7 +157,9 @@ public: // LEGACY //! find list of translations that can translates a portion of src. Used by confusion network decoding - virtual const TargetPhraseCollectionWithSourcePhrase* GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const& range) const; + virtual + TargetPhraseCollectionWithSourcePhrase::shared_ptr + GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const& range) const; protected: static std::vector<PhraseDictionary*> s_staticColl; @@ -184,7 +186,10 @@ protected: mutable boost::scoped_ptr<CacheColl> m_cache; #endif - virtual const TargetPhraseCollection *GetTargetPhraseCollectionNonCacheLEGACY(const Phrase& src) const; + virtual + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionNonCacheLEGACY(const Phrase& src) const; + void ReduceCache() const; protected: diff --git a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp index 5fc569c90..11346f0b6 100644 --- a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp +++ b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp @@ -150,15 +150,15 @@ void PhraseDictionaryDynamicCacheBased::InitializeForInput(ttasksptr const& ttas ReduceCache(); } -const TargetPhraseCollection *PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const +TargetPhraseCollection::shared_ptr PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const { #ifdef WITH_THREADS boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock); #endif - TargetPhraseCollection* tpc = NULL; + TargetPhraseCollection::shared_ptr tpc; cacheMap::const_iterator it = m_cacheTM.find(source); if(it != m_cacheTM.end()) { - tpc = new TargetPhraseCollection(*(it->second).first); + tpc.reset(new TargetPhraseCollection(*(it->second).first)); std::vector<const TargetPhrase*>::const_iterator it2 = tpc->begin(); @@ -174,15 +174,15 @@ const TargetPhraseCollection *PhraseDictionaryDynamicCacheBased::GetTargetPhrase return tpc; } -const TargetPhraseCollection* PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollectionLEGACY(Phrase const &src) const +TargetPhraseCollection::shared_ptr PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollectionLEGACY(Phrase const &src) const { - const TargetPhraseCollection *ret = GetTargetPhraseCollection(src); + TargetPhraseCollection::shared_ptr ret = GetTargetPhraseCollection(src); return ret; } -const TargetPhraseCollection* PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const +TargetPhraseCollection::shared_ptr PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const { - const TargetPhraseCollection *ret = GetTargetPhraseCollection(src); + TargetPhraseCollection::shared_ptr ret = GetTargetPhraseCollection(src); return ret; } @@ -366,7 +366,7 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(Phrase sp, Phrase tp) // and then add new entry TargetCollectionAgePair TgtCollAgePair = it->second; - TargetPhraseCollection* tpc = TgtCollAgePair.first; + TargetPhraseCollection::shared_ptr tpc = TgtCollAgePair.first; AgeCollection* ac = TgtCollAgePair.second; const Phrase* p_ptr = NULL; TargetPhrase* tp_ptr = NULL; @@ -397,7 +397,7 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(Phrase sp, Phrase tp) if (tpc->GetSize() == 0) { // delete the entry from m_cacheTM in case it points to an empty TargetPhraseCollection and AgeCollection ac->clear(); - delete tpc; + tpc.reset(); delete ac; m_cacheTM.erase(sp); } @@ -451,14 +451,14 @@ void PhraseDictionaryDynamicCacheBased::ClearSource(Phrase sp) //sp is found TargetCollectionAgePair TgtCollAgePair = it->second; - TargetPhraseCollection* tpc = TgtCollAgePair.first; + TargetPhraseCollection::shared_ptr tpc = TgtCollAgePair.first; AgeCollection* ac = TgtCollAgePair.second; m_entries-=tpc->GetSize(); //reduce the total amount of entries of the cache // delete the entry from m_cacheTM in case it points to an empty TargetPhraseCollection and AgeCollection ac->clear(); - delete tpc; + tpc.reset(); delete ac; m_cacheTM.erase(sp); } else { @@ -558,7 +558,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, TargetPhrase tp, int a // and then add new entry TargetCollectionAgePair TgtCollAgePair = it->second; - TargetPhraseCollection* tpc = TgtCollAgePair.first; + TargetPhraseCollection::shared_ptr tpc = TgtCollAgePair.first; AgeCollection* ac = TgtCollAgePair.second; // const TargetPhrase* p_ptr = NULL; const Phrase* p_ptr = NULL; @@ -599,7 +599,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, TargetPhrase tp, int a // create target collection // we have to create new target collection age pair and add new entry to target collection age pair - TargetPhraseCollection* tpc = new TargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpc(new TargetPhraseCollection); AgeCollection* ac = new AgeCollection(); m_cacheTM.insert(make_pair(sp,make_pair(tpc,ac))); @@ -629,13 +629,13 @@ void PhraseDictionaryDynamicCacheBased::Decay() void PhraseDictionaryDynamicCacheBased::Decay(Phrase sp) { VERBOSE(3,"void PhraseDictionaryDynamicCacheBased::Decay(Phrase sp) sp:|" << sp << "|" << std::endl); - cacheMap::const_iterator it = m_cacheTM.find(sp); + cacheMap::iterator it = m_cacheTM.find(sp); if (it != m_cacheTM.end()) { VERBOSE(3,"found:|" << sp << "|" << std::endl); //sp is found TargetCollectionAgePair TgtCollAgePair = it->second; - TargetPhraseCollection* tpc = TgtCollAgePair.first; + TargetPhraseCollection::shared_ptr tpc = TgtCollAgePair.first; AgeCollection* ac = TgtCollAgePair.second; //loop in inverted order to allow a correct deletion of std::vectors tpc and ac @@ -661,7 +661,7 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase sp) // delete the entry from m_cacheTM in case it points to an empty TargetPhraseCollection and AgeCollection (((*it).second).second)->clear(); delete ((*it).second).second; - delete ((*it).second).first; + ((*it).second).first.reset(); m_cacheTM.erase(sp); } } else { @@ -703,11 +703,11 @@ void PhraseDictionaryDynamicCacheBased::Clear() #ifdef WITH_THREADS boost::shared_lock<boost::shared_mutex> lock(m_cacheLock); #endif - cacheMap::const_iterator it; + cacheMap::iterator it; for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) { (((*it).second).second)->clear(); delete ((*it).second).second; - delete ((*it).second).first; + ((*it).second).first.reset(); } m_cacheTM.clear(); m_entries = 0; @@ -746,7 +746,7 @@ void PhraseDictionaryDynamicCacheBased::Print() const cacheMap::const_iterator it; for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) { std::string source = (it->first).ToString(); - TargetPhraseCollection* tpc = (it->second).first; + TargetPhraseCollection::shared_ptr tpc = (it->second).first; TargetPhraseCollection::iterator itr; for(itr = tpc->begin(); itr != tpc->end(); itr++) { std::string target = (*itr)->ToString(); diff --git a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h index 8aee16051..44488d719 100644 --- a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h +++ b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h @@ -53,7 +53,7 @@ class PhraseDictionaryDynamicCacheBased : public PhraseDictionary { typedef std::vector<unsigned int> AgeCollection; - typedef std::pair<TargetPhraseCollection*, AgeCollection*> TargetCollectionAgePair; + typedef std::pair<TargetPhraseCollection::shared_ptr , AgeCollection*> TargetCollectionAgePair; typedef std::map<Phrase, TargetCollectionAgePair> cacheMap; // data structure for the cache @@ -111,9 +111,14 @@ public: void Load(); void Load(const std::string files); - const TargetPhraseCollection* GetTargetPhraseCollection(const Phrase &src) const; - const TargetPhraseCollection* GetTargetPhraseCollectionLEGACY(Phrase const &src) const; - const TargetPhraseCollection* GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const; + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection(const Phrase &src) const; + + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionLEGACY(Phrase const &src) const; + + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const; // for phrase-based model // void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; diff --git a/moses/TranslationModel/PhraseDictionaryGroup.cpp b/moses/TranslationModel/PhraseDictionaryGroup.cpp new file mode 100644 index 000000000..b672d09e7 --- /dev/null +++ b/moses/TranslationModel/PhraseDictionaryGroup.cpp @@ -0,0 +1,217 @@ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include "moses/TranslationModel/PhraseDictionaryGroup.h" + +#include <boost/foreach.hpp> +#include <boost/unordered_map.hpp> + +#include "util/exception.hh" + +using namespace std; +using namespace boost; + +namespace Moses +{ + +PhraseDictionaryGroup::PhraseDictionaryGroup(const string &line) + : PhraseDictionary(line, true), + m_numModels(0), + m_restrict(false) +{ + ReadParameters(); +} + +void PhraseDictionaryGroup::SetParameter(const string& key, const string& value) +{ + if (key == "members") { + m_memberPDStrs = Tokenize(value, ","); + m_numModels = m_memberPDStrs.size(); + } else if (key == "restrict") { + m_restrict = Scan<bool>(value); + } else { + PhraseDictionary::SetParameter(key, value); + } +} + +void PhraseDictionaryGroup::Load() +{ + SetFeaturesToApply(); + m_pdFeature.push_back(const_cast<PhraseDictionaryGroup*>(this)); + + // Locate/check component phrase tables + size_t componentWeights = 0; + BOOST_FOREACH(const string& pdName, m_memberPDStrs) { + bool pdFound = false; + BOOST_FOREACH(PhraseDictionary* pd, PhraseDictionary::GetColl()) { + if (pd->GetScoreProducerDescription() == pdName) { + pdFound = true; + m_memberPDs.push_back(pd); + componentWeights += pd->GetNumScoreComponents(); + } + } + UTIL_THROW_IF2(!pdFound, + "Could not find component phrase table " << pdName); + } + UTIL_THROW_IF2(componentWeights != m_numScoreComponents, + "Total number of component model scores is unequal to specified number of scores"); +} + +void PhraseDictionaryGroup::GetTargetPhraseCollectionBatch( + const ttasksptr& ttask, const InputPathList& inputPathQueue) const +{ + // Some implementations (mmsapt) do work in PrefixExists + BOOST_FOREACH(const InputPath* inputPath, inputPathQueue) { + const Phrase& phrase = inputPath->GetPhrase(); + BOOST_FOREACH(const PhraseDictionary* pd, m_memberPDs) { + pd->PrefixExists(ttask, phrase); + } + } + // Look up each input in each model + BOOST_FOREACH(InputPath* inputPath, inputPathQueue) { + const Phrase &phrase = inputPath->GetPhrase(); + TargetPhraseCollection::shared_ptr targetPhrases = + this->GetTargetPhraseCollectionLEGACY(ttask, phrase); + inputPath->SetTargetPhrases(*this, targetPhrases, NULL); + } +} + +TargetPhraseCollection::shared_ptr PhraseDictionaryGroup::GetTargetPhraseCollectionLEGACY( + const Phrase& src) const +{ + UTIL_THROW2("Don't call me without the translation task."); +} + +TargetPhraseCollection::shared_ptr +PhraseDictionaryGroup:: +GetTargetPhraseCollectionLEGACY(const ttasksptr& ttask, const Phrase& src) const +{ + TargetPhraseCollection::shared_ptr ret + = CreateTargetPhraseCollection(ttask, src); + ret->NthElement(m_tableLimit); // sort the phrases for pruning later + const_cast<PhraseDictionaryGroup*>(this)->CacheForCleanup(ret); + return ret; +} + +TargetPhraseCollection::shared_ptr +PhraseDictionaryGroup:: +CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const +{ + // Aggregation of phrases and the scores that will be applied to them + vector<TargetPhrase*> allPhrases; + unordered_map<const TargetPhrase*, vector<float>, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > allScores; + + // For each model + size_t offset = 0; + for (size_t i = 0; i < m_numModels; ++i) { + + // Collect phrases from this table + const PhraseDictionary& pd = *m_memberPDs[i]; + TargetPhraseCollection::shared_ptr + ret_raw = pd.GetTargetPhraseCollectionLEGACY(ttask, src); + + if (ret_raw != NULL) { + // Process each phrase from table + BOOST_FOREACH(const TargetPhrase* targetPhrase, *ret_raw) { + vector<float> raw_scores = + targetPhrase->GetScoreBreakdown().GetScoresForProducer(&pd); + + // Phrase not in collection -> add if unrestricted or first model + if (allScores.find(targetPhrase) == allScores.end()) { + if (m_restrict && i > 0) { + continue; + } + + // Copy phrase to avoid disrupting base model + TargetPhrase* phrase = new TargetPhrase(*targetPhrase); + // Correct future cost estimates and total score + phrase->GetScoreBreakdown().InvertDenseFeatures(&pd); + vector<FeatureFunction*> pd_feature; + pd_feature.push_back(m_memberPDs[i]); + const vector<FeatureFunction*> pd_feature_const(pd_feature); + phrase->EvaluateInIsolation(src, pd_feature_const); + // Zero out scores from original phrase table + phrase->GetScoreBreakdown().ZeroDenseFeatures(&pd); + // Add phrase entry + allPhrases.push_back(phrase); + allScores[targetPhrase] = vector<float>(m_numScoreComponents, 0); + } + vector<float>& scores = allScores.find(targetPhrase)->second; + + // Copy scores from this model + for (size_t j = 0; j < pd.GetNumScoreComponents(); ++j) { + scores[offset + j] = raw_scores[j]; + } + } + } + offset += pd.GetNumScoreComponents(); + } + + // Apply scores to phrases and add them to return collection + TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection); + const vector<FeatureFunction*> pd_feature_const(m_pdFeature); + BOOST_FOREACH(TargetPhrase* phrase, allPhrases) { + phrase->GetScoreBreakdown().Assign(this, allScores.find(phrase)->second); + // Correct future cost estimates and total score + phrase->EvaluateInIsolation(src, pd_feature_const); + ret->Add(phrase); + } + + return ret; +} + +ChartRuleLookupManager* +PhraseDictionaryGroup:: +CreateRuleLookupManager(const ChartParser &, + const ChartCellCollectionBase&, size_t) +{ + UTIL_THROW(util::Exception, "Phrase table used in chart decoder"); +} + +//copied from PhraseDictionaryCompact; free memory allocated to TargetPhraseCollection (and each TargetPhrase) at end of sentence +void PhraseDictionaryGroup::CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) +{ + PhraseCache &ref = GetPhraseCache(); + ref.push_back(tpc); +} + +void +PhraseDictionaryGroup:: +CleanUpAfterSentenceProcessing(const InputType &source) +{ + GetPhraseCache().clear(); + // PhraseCache &ref = GetPhraseCache(); + // for (PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) { + // delete *it; + // } + + // PhraseCache temp; + // temp.swap(ref); + + CleanUpComponentModels(source); +} + +void PhraseDictionaryGroup::CleanUpComponentModels(const InputType &source) +{ + for (size_t i = 0; i < m_numModels; ++i) { + m_memberPDs[i]->CleanUpAfterSentenceProcessing(source); + } +} + +} //namespace diff --git a/moses/TranslationModel/PhraseDictionaryGroup.h b/moses/TranslationModel/PhraseDictionaryGroup.h new file mode 100644 index 000000000..f8deca41f --- /dev/null +++ b/moses/TranslationModel/PhraseDictionaryGroup.h @@ -0,0 +1,104 @@ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_PhraseDictionaryGroup_h +#define moses_PhraseDictionaryGroup_h + +#include "moses/TranslationModel/PhraseDictionary.h" + +#include <boost/unordered_map.hpp> +#include <boost/thread/shared_mutex.hpp> +#include "moses/StaticData.h" +#include "moses/TargetPhrase.h" +#include "moses/Util.h" + +namespace Moses +{ + +/** Combines multiple phrase tables into a single interface. Each member phrase + * table scores each phrase and a single set of translations/scores is returned. + * If a phrase is not in one of the tables, its scores are zero-filled. Use the + * "restrict" option to restrict phrases to those in the table-limit of the + * first member table, intended to be a "union" table built on all data. + */ +class PhraseDictionaryGroup: public PhraseDictionary +{ + +public: + PhraseDictionaryGroup(const std::string& line); + void Load(); + TargetPhraseCollection::shared_ptr + CreateTargetPhraseCollection(const ttasksptr& ttask, + const Phrase& src) const; + std::vector<std::vector<float> > getWeights(size_t numWeights, + bool normalize) const; + void CacheForCleanup(TargetPhraseCollection::shared_ptr tpc); + void CleanUpAfterSentenceProcessing(const InputType& source); + void CleanUpComponentModels(const InputType& source); + // functions below override the base class + void GetTargetPhraseCollectionBatch(const ttasksptr& ttask, + const InputPathList &inputPathQueue) const; + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionLEGACY( + const Phrase& src) const; + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionLEGACY( + const ttasksptr& ttask, const Phrase& src) const; + void InitializeForInput(ttasksptr const& ttask) { + /* Don't do anything source specific here as this object is shared between threads.*/ + } + ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, + const ChartCellCollectionBase&, std::size_t); + void SetParameter(const std::string& key, const std::string& value); + +protected: + std::vector<std::string> m_memberPDStrs; + std::vector<PhraseDictionary*> m_memberPDs; + size_t m_numModels; + bool m_restrict; + std::vector<FeatureFunction*> m_pdFeature; + + typedef std::vector<TargetPhraseCollection::shared_ptr > PhraseCache; +#ifdef WITH_THREADS + boost::shared_mutex m_lock_cache; + typedef std::map<boost::thread::id, PhraseCache> SentenceCache; +#else + typedef PhraseCache SentenceCache; +#endif + SentenceCache m_sentenceCache; + + PhraseCache& GetPhraseCache() { +#ifdef WITH_THREADS + { + // first try read-only lock + boost::shared_lock<boost::shared_mutex> read_lock(m_lock_cache); + SentenceCache::iterator i = m_sentenceCache.find( + boost::this_thread::get_id()); + if (i != m_sentenceCache.end()) + return i->second; + } + boost::unique_lock<boost::shared_mutex> lock(m_lock_cache); + return m_sentenceCache[boost::this_thread::get_id()]; +#else + return m_sentenceCache; +#endif + } +}; + +} // end namespace + +#endif diff --git a/moses/TranslationModel/PhraseDictionaryMemory.cpp b/moses/TranslationModel/PhraseDictionaryMemory.cpp index 1724748bd..3053e46db 100644 --- a/moses/TranslationModel/PhraseDictionaryMemory.cpp +++ b/moses/TranslationModel/PhraseDictionaryMemory.cpp @@ -49,16 +49,17 @@ PhraseDictionaryMemory::PhraseDictionaryMemory(const std::string &line) } -TargetPhraseCollection &PhraseDictionaryMemory::GetOrCreateTargetPhraseCollection( - const Phrase &source - , const TargetPhrase &target - , const Word *sourceLHS) +TargetPhraseCollection::shared_ptr +PhraseDictionaryMemory:: +GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) { PhraseDictionaryNodeMemory &currNode = GetOrCreateNode(source, target, sourceLHS); return currNode.GetTargetPhraseCollection(); } -const TargetPhraseCollection* +TargetPhraseCollection::shared_ptr PhraseDictionaryMemory:: GetTargetPhraseCollectionLEGACY(const Phrase& sourceOrig) const { @@ -73,10 +74,10 @@ GetTargetPhraseCollectionLEGACY(const Phrase& sourceOrig) const const Word& word = source.GetWord(pos); currNode = currNode->GetChild(word); if (currNode == NULL) - return NULL; + return TargetPhraseCollection::shared_ptr(); } - return &currNode->GetTargetPhraseCollection(); + return currNode->GetTargetPhraseCollection(); } PhraseDictionaryNodeMemory &PhraseDictionaryMemory::GetOrCreateNode(const Phrase &source @@ -168,12 +169,11 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const lastWord.OnlyTheseFactors(m_inputFactors); const PhraseDictionaryNodeMemory *ptNode = prevPtNode->GetChild(lastWord); + TargetPhraseCollection::shared_ptr targetPhrases; if (ptNode) { - const TargetPhraseCollection &targetPhrases = ptNode->GetTargetPhraseCollection(); - inputPath.SetTargetPhrases(*this, &targetPhrases, ptNode); - } else { - inputPath.SetTargetPhrases(*this, NULL, NULL); + targetPhrases = ptNode->GetTargetPhraseCollection(); } + inputPath.SetTargetPhrases(*this, targetPhrases, ptNode); } } } diff --git a/moses/TranslationModel/PhraseDictionaryMemory.h b/moses/TranslationModel/PhraseDictionaryMemory.h index 723beaea2..86bda5087 100644 --- a/moses/TranslationModel/PhraseDictionaryMemory.h +++ b/moses/TranslationModel/PhraseDictionaryMemory.h @@ -56,18 +56,22 @@ public: std::size_t); // only used by multi-model phrase table, and other meta-features - const TargetPhraseCollection *GetTargetPhraseCollectionLEGACY(const Phrase& src) const; - void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionLEGACY(const Phrase& src) const; + + void + GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; TO_STRING(); protected: - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection + (const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); - PhraseDictionaryNodeMemory &GetOrCreateNode(const Phrase &source - , const TargetPhrase &target - , const Word *sourceLHS); + PhraseDictionaryNodeMemory & + GetOrCreateNode(const Phrase &source, const TargetPhrase &target, + const Word *sourceLHS); void SortAndPrune(); diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp index 4b4df0a02..e8ffa9d1d 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp @@ -17,6 +17,7 @@ License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ #include "util/exception.hh" +#include "util/string_stream.hh" #include "moses/TranslationModel/PhraseDictionaryMultiModel.h" @@ -25,8 +26,10 @@ using namespace std; namespace Moses { -PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line) - :PhraseDictionary(line, true) + +PhraseDictionaryMultiModel:: +PhraseDictionaryMultiModel(const std::string &line) + : PhraseDictionary(line, true) { ReadParameters(); @@ -36,24 +39,16 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line) m_pdStr.size()*numWeights != m_multimodelweights.size(), "Number of scores and weights are not equal"); } else if (m_mode == "all" || m_mode == "all-restrict") { - size_t componentWeights = 0; - for(size_t i = 0; i < m_numModels; ++i) { - const string &ptName = m_pdStr[i]; - PhraseDictionary *pt = FindPhraseDictionary(ptName); - UTIL_THROW_IF2(pt == NULL, - "Could not find component phrase table " << ptName); - componentWeights += pt->GetNumScoreComponents(); - } - UTIL_THROW_IF2(componentWeights != m_numScoreComponents, - "Total number of component model scores is unequal to specified number of scores"); + UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false"); } else { - ostringstream msg; + util::StringStream msg; msg << "combination mode unknown: " << m_mode; throw runtime_error(msg.str()); } } -PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::string &line) +PhraseDictionaryMultiModel:: +PhraseDictionaryMultiModel(int type, const std::string &line) :PhraseDictionary(line, true) { if (type == 1) { @@ -64,7 +59,9 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::stri } } -void PhraseDictionaryMultiModel::SetParameter(const std::string& key, const std::string& value) +void +PhraseDictionaryMultiModel:: +SetParameter(const std::string& key, const std::string& value) { if (key == "mode") { m_mode = value; @@ -78,9 +75,9 @@ void PhraseDictionaryMultiModel::SetParameter(const std::string& key, const std: } } -PhraseDictionaryMultiModel::~PhraseDictionaryMultiModel() -{ -} +PhraseDictionaryMultiModel:: +~PhraseDictionaryMultiModel() +{ } void PhraseDictionaryMultiModel::Load() { @@ -96,29 +93,21 @@ void PhraseDictionaryMultiModel::Load() } } - -const TargetPhraseCollection *PhraseDictionaryMultiModel::GetTargetPhraseCollectionLEGACY(const Phrase& src) const +TargetPhraseCollection::shared_ptr +PhraseDictionaryMultiModel:: +GetTargetPhraseCollectionLEGACY(const Phrase& src) const { std::vector<std::vector<float> > multimodelweights; + multimodelweights = getWeights(m_numScoreComponents, true); + TargetPhraseCollection::shared_ptr ret; - if (m_mode == "interpolate") { - multimodelweights = getWeights(m_numScoreComponents, true); - } - - TargetPhraseCollection *ret = NULL; - - if (m_mode == "interpolate") { - std::map<std::string,multiModelStatistics*>* allStats = new(std::map<std::string,multiModelStatistics*>); - CollectSufficientStatistics(src, allStats); - ret = CreateTargetPhraseCollectionLinearInterpolation(src, allStats, multimodelweights); - RemoveAllInMap(*allStats); - delete allStats; - } else if (m_mode == "all") { - ret = CreateTargetPhraseCollectionAll(src, false); - } else if (m_mode == "all-restrict") { - ret = CreateTargetPhraseCollectionAll(src, true); - } + std::map<std::string, multiModelStats*>* allStats; + allStats = new(std::map<std::string,multiModelStats*>); + CollectSufficientStatistics(src, allStats); + ret = CreateTargetPhraseCollectionLinearInterpolation(src, allStats, multimodelweights); + RemoveAllInMap(*allStats); + delete allStats; // ??? Why the detour through malloc? UG ret->NthElement(m_tableLimit); // sort the phrases for pruning later const_cast<PhraseDictionaryMultiModel*>(this)->CacheForCleanup(ret); @@ -126,16 +115,19 @@ const TargetPhraseCollection *PhraseDictionaryMultiModel::GetTargetPhraseCollect return ret; } - -void PhraseDictionaryMultiModel::CollectSufficientStatistics(const Phrase& src, std::map<std::string,multiModelStatistics*>* allStats) const +void +PhraseDictionaryMultiModel:: +CollectSufficientStatistics +(const Phrase& src, std::map<std::string, multiModelStats*>* allStats) const { for(size_t i = 0; i < m_numModels; ++i) { const PhraseDictionary &pd = *m_pd[i]; - TargetPhraseCollection *ret_raw = (TargetPhraseCollection*) pd.GetTargetPhraseCollectionLEGACY( src); + TargetPhraseCollection::shared_ptr ret_raw; + ret_raw = pd.GetTargetPhraseCollectionLEGACY(src); if (ret_raw != NULL) { - TargetPhraseCollection::iterator iterTargetPhrase, iterLast; + TargetPhraseCollection::const_iterator iterTargetPhrase, iterLast; if (m_tableLimit != 0 && ret_raw->GetSize() > m_tableLimit) { iterLast = ret_raw->begin() + m_tableLimit; } else { @@ -149,7 +141,7 @@ void PhraseDictionaryMultiModel::CollectSufficientStatistics(const Phrase& src, std::string targetString = targetPhrase->GetStringRep(m_output); if (allStats->find(targetString) == allStats->end()) { - multiModelStatistics * statistics = new multiModelStatistics; + multiModelStats * statistics = new multiModelStats; statistics->targetPhrase = new TargetPhrase(*targetPhrase); //make a copy so that we don't overwrite the original phrase table info statistics->p.resize(m_numScoreComponents); for(size_t j = 0; j < m_numScoreComponents; ++j) { @@ -168,7 +160,7 @@ void PhraseDictionaryMultiModel::CollectSufficientStatistics(const Phrase& src, (*allStats)[targetString] = statistics; } - multiModelStatistics * statistics = (*allStats)[targetString]; + multiModelStats * statistics = (*allStats)[targetString]; for(size_t j = 0; j < m_numScoreComponents; ++j) { statistics->p[j][i] = UntransformScore(raw_scores[j]); @@ -180,12 +172,17 @@ void PhraseDictionaryMultiModel::CollectSufficientStatistics(const Phrase& src, } } -TargetPhraseCollection* PhraseDictionaryMultiModel::CreateTargetPhraseCollectionLinearInterpolation(const Phrase& src, std::map<std::string,multiModelStatistics*>* allStats, std::vector<std::vector<float> > &multimodelweights) const +TargetPhraseCollection::shared_ptr +PhraseDictionaryMultiModel:: +CreateTargetPhraseCollectionLinearInterpolation +( const Phrase& src, + std::map<std::string,multiModelStats*>* allStats, + std::vector<std::vector<float> > &multimodelweights) const { - TargetPhraseCollection *ret = new TargetPhraseCollection(); - for ( std::map< std::string, multiModelStatistics*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { + TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection); + for ( std::map< std::string, multiModelStats*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { - multiModelStatistics * statistics = iter->second; + multiModelStats * statistics = iter->second; Scores scoreVector(m_numScoreComponents); @@ -206,91 +203,10 @@ TargetPhraseCollection* PhraseDictionaryMultiModel::CreateTargetPhraseCollection return ret; } -TargetPhraseCollection* PhraseDictionaryMultiModel::CreateTargetPhraseCollectionAll(const Phrase& src, const bool restricted) const -{ - // Collect phrases from all models - std::map<std::string, multiModelPhrase*> allPhrases; - size_t offset = 0; - for(size_t i = 0; i < m_numModels; ++i) { - const PhraseDictionary &pd = *m_pd[i]; - - TargetPhraseCollection *ret_raw = (TargetPhraseCollection*) pd.GetTargetPhraseCollectionLEGACY(src); - if (ret_raw != NULL) { - - TargetPhraseCollection::iterator iterTargetPhrase, iterLast; - if (m_tableLimit != 0 && ret_raw->GetSize() > m_tableLimit) { - iterLast = ret_raw->begin() + m_tableLimit; - } else { - iterLast = ret_raw->end(); - } - - for (iterTargetPhrase = ret_raw->begin(); iterTargetPhrase != iterLast; ++iterTargetPhrase) { - const TargetPhrase* targetPhrase = *iterTargetPhrase; - std::vector<float> raw_scores = targetPhrase->GetScoreBreakdown().GetScoresForProducer(&pd); - - std::string targetString = targetPhrase->GetStringRep(m_output); - // Phrase not in collection -> add if unrestricted (all) or first model (all-restrict) - if (allPhrases.find(targetString) == allPhrases.end()) { - // all-restrict and not first model: skip adding unseen phrase - if (restricted && i > 0) { - continue; - } - - multiModelPhrase* phrase = new multiModelPhrase; - phrase->targetPhrase = new TargetPhrase(*targetPhrase); //make a copy so that we don't overwrite the original phrase table info - // p contains scores from all models in order. Values default to zero for models that do not contain phrase. - phrase->p.resize(m_numScoreComponents, 0); - - //correct future cost estimates and total score - phrase->targetPhrase->GetScoreBreakdown().InvertDenseFeatures(&pd); - vector<FeatureFunction*> pd_feature; - pd_feature.push_back(m_pd[i]); - const vector<FeatureFunction*> pd_feature_const(pd_feature); - phrase->targetPhrase->EvaluateInIsolation(src, pd_feature_const); - // zero out scores from original phrase table - phrase->targetPhrase->GetScoreBreakdown().ZeroDenseFeatures(&pd); - - allPhrases[targetString] = phrase; - - } - multiModelPhrase* phrase = allPhrases[targetString]; - - for(size_t j = 0; j < pd.GetNumScoreComponents(); ++j) { - phrase->p[offset + j] = raw_scores[j]; - } - } - } - offset += pd.GetNumScoreComponents(); - } - - // Copy accumulated score vectors to phrases - TargetPhraseCollection* ret = new TargetPhraseCollection(); - for (std::map<std::string, multiModelPhrase*>::const_iterator iter = allPhrases.begin(); iter != allPhrases.end(); ++iter) { - - multiModelPhrase* phrase = iter->second; - Scores scoreVector(m_numScoreComponents); - - for(size_t i = 0; i < m_numScoreComponents; ++i) { - scoreVector[i] = phrase->p[i]; - } - - phrase->targetPhrase->GetScoreBreakdown().Assign(this, scoreVector); - - //correct future cost estimates and total score - vector<FeatureFunction*> pd_feature; - pd_feature.push_back(const_cast<PhraseDictionaryMultiModel*>(this)); - const vector<FeatureFunction*> pd_feature_const(pd_feature); - phrase->targetPhrase->EvaluateInIsolation(src, pd_feature_const); - - ret->Add(new TargetPhrase(*phrase->targetPhrase)); - } - - RemoveAllInMap(allPhrases); - return ret; -} - //TODO: is it worth caching the results as long as weights don't change? -std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t numWeights, bool normalize) const +std::vector<std::vector<float> > +PhraseDictionaryMultiModel:: +getWeights(size_t numWeights, bool normalize) const { const std::vector<float>* weights_ptr; std::vector<float> raw_weights; @@ -313,7 +229,7 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n raw_weights.push_back(1.0/m_numModels); //uniform weights created online } } else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) { - std::stringstream strme; + util::StringStream strme; strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << "."; UTIL_THROW(util::Exception, strme.str()); } else { @@ -339,7 +255,9 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n return multimodelweights; } -std::vector<float> PhraseDictionaryMultiModel::normalizeWeights(std::vector<float> &weights) const +std::vector<float> +PhraseDictionaryMultiModel:: +normalizeWeights(std::vector<float> &weights) const { std::vector<float> ret (m_numModels); float total = std::accumulate(weights.begin(),weights.end(),0.0); @@ -350,29 +268,36 @@ std::vector<float> PhraseDictionaryMultiModel::normalizeWeights(std::vector<floa } -ChartRuleLookupManager *PhraseDictionaryMultiModel::CreateRuleLookupManager(const ChartParser &, const ChartCellCollectionBase&, std::size_t) +ChartRuleLookupManager * +PhraseDictionaryMultiModel:: +CreateRuleLookupManager(const ChartParser &, const ChartCellCollectionBase&, + std::size_t) { UTIL_THROW(util::Exception, "Phrase table used in chart decoder"); } //copied from PhraseDictionaryCompact; free memory allocated to TargetPhraseCollection (and each TargetPhrase) at end of sentence -void PhraseDictionaryMultiModel::CacheForCleanup(TargetPhraseCollection* tpc) +void +PhraseDictionaryMultiModel:: +CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) { - PhraseCache &ref = GetPhraseCache(); - ref.push_back(tpc); + GetPhraseCache().push_back(tpc); } -void PhraseDictionaryMultiModel::CleanUpAfterSentenceProcessing(const InputType &source) +void +PhraseDictionaryMultiModel:: +CleanUpAfterSentenceProcessing(const InputType &source) { - PhraseCache &ref = GetPhraseCache(); - for(PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) { - delete *it; - } + // PhraseCache &ref = GetPhraseCache(); + // for(PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) { + // it->reset(); + // } - PhraseCache temp; - temp.swap(ref); + // PhraseCache temp; + // temp.swap(ref); + GetPhraseCache().clear(); CleanUpComponentModels(source); @@ -381,14 +306,18 @@ void PhraseDictionaryMultiModel::CleanUpAfterSentenceProcessing(const InputType } -void PhraseDictionaryMultiModel::CleanUpComponentModels(const InputType &source) +void +PhraseDictionaryMultiModel:: +CleanUpComponentModels(const InputType &source) { for(size_t i = 0; i < m_numModels; ++i) { m_pd[i]->CleanUpAfterSentenceProcessing(source); } } -const std::vector<float>* PhraseDictionaryMultiModel::GetTemporaryMultiModelWeightsVector() const +const std::vector<float>* +PhraseDictionaryMultiModel:: +GetTemporaryMultiModelWeightsVector() const { #ifdef WITH_THREADS boost::shared_lock<boost::shared_mutex> read_lock(m_lock_weights); @@ -402,7 +331,9 @@ const std::vector<float>* PhraseDictionaryMultiModel::GetTemporaryMultiModelWeig #endif } -void PhraseDictionaryMultiModel::SetTemporaryMultiModelWeightsVector(std::vector<float> weights) +void +PhraseDictionaryMultiModel:: +SetTemporaryMultiModelWeightsVector(std::vector<float> weights) { #ifdef WITH_THREADS boost::unique_lock<boost::shared_mutex> lock(m_lock_weights); @@ -413,7 +344,9 @@ void PhraseDictionaryMultiModel::SetTemporaryMultiModelWeightsVector(std::vector } #ifdef WITH_DLIB -vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, string> > &phrase_pair_vector) +vector<float> +PhraseDictionaryMultiModel:: +MinimizePerplexity(vector<pair<string, string> > &phrase_pair_vector) { map<pair<string, string>, size_t> phrase_pair_map; @@ -422,7 +355,7 @@ vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, phrase_pair_map[*iter] += 1; } - vector<multiModelStatisticsOptimization*> optimizerStats; + vector<multiModelStatsOptimization*> optimizerStats; for ( map<pair<string, string>, size_t>::iterator iter = phrase_pair_map.begin(); iter != phrase_pair_map.end(); ++iter ) { @@ -431,7 +364,7 @@ vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, string target_string = phrase_pair.second; vector<float> fs(m_numModels); - map<string,multiModelStatistics*>* allStats = new(map<string,multiModelStatistics*>); + map<string,multiModelStats*>* allStats = new(map<string,multiModelStats*>); Phrase sourcePhrase(0); sourcePhrase.CreateFromString(Input, m_input, source_string, NULL); @@ -445,7 +378,7 @@ vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, continue; } - multiModelStatisticsOptimization* targetStatistics = new multiModelStatisticsOptimization(); + multiModelStatsOptimization* targetStatistics = new multiModelStatsOptimization(); targetStatistics->targetPhrase = new TargetPhrase(*(*allStats)[target_string]->targetPhrase); targetStatistics->p = (*allStats)[target_string]->p; targetStatistics->f = iter->second; @@ -485,7 +418,9 @@ vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, } -vector<float> PhraseDictionaryMultiModel::Optimize(OptimizationObjective *ObjectiveFunction, size_t numModels) +vector<float> +PhraseDictionaryMultiModel:: +Optimize(OptimizationObjective *ObjectiveFunction, size_t numModels) { dlib::matrix<double,0,1> starting_point; @@ -530,8 +465,8 @@ double CrossEntropy::operator() ( const dlib::matrix<double,0,1>& arg) const weight_vector = m_model->normalizeWeights(weight_vector); } - for ( std::vector<multiModelStatisticsOptimization*>::const_iterator iter = m_optimizerStats.begin(); iter != m_optimizerStats.end(); ++iter ) { - multiModelStatisticsOptimization* statistics = *iter; + for ( std::vector<multiModelStatsOptimization*>::const_iterator iter = m_optimizerStats.begin(); iter != m_optimizerStats.end(); ++iter ) { + multiModelStatsOptimization* statistics = *iter; size_t f = statistics->f; double score; diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.h b/moses/TranslationModel/PhraseDictionaryMultiModel.h index eb3152c3c..bdb399ddb 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModel.h +++ b/moses/TranslationModel/PhraseDictionaryMultiModel.h @@ -36,15 +36,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -struct multiModelStatistics { +struct multiModelStats { TargetPhrase *targetPhrase; std::vector<std::vector<float> > p; - ~multiModelStatistics() { + ~multiModelStats() { delete targetPhrase; }; }; -struct multiModelStatisticsOptimization: multiModelStatistics { +struct multiModelStatsOptimization: multiModelStats { size_t f; }; @@ -71,28 +71,59 @@ public: PhraseDictionaryMultiModel(int type, const std::string &line); ~PhraseDictionaryMultiModel(); void Load(); - virtual void CollectSufficientStatistics(const Phrase& src, std::map<std::string,multiModelStatistics*>* allStats) const; - virtual TargetPhraseCollection* CreateTargetPhraseCollectionLinearInterpolation(const Phrase& src, std::map<std::string,multiModelStatistics*>* allStats, std::vector<std::vector<float> > &multimodelweights) const; - virtual TargetPhraseCollection* CreateTargetPhraseCollectionAll(const Phrase& src, const bool restricted = false) const; - std::vector<std::vector<float> > getWeights(size_t numWeights, bool normalize) const; - std::vector<float> normalizeWeights(std::vector<float> &weights) const; - void CacheForCleanup(TargetPhraseCollection* tpc); - void CleanUpAfterSentenceProcessing(const InputType &source); - virtual void CleanUpComponentModels(const InputType &source); + + virtual void + CollectSufficientStatistics + (const Phrase& src, std::map<std::string,multiModelStats*>* allStats) + const; + + virtual TargetPhraseCollection::shared_ptr + CreateTargetPhraseCollectionLinearInterpolation + (const Phrase& src, std::map<std::string,multiModelStats*>* allStats, + std::vector<std::vector<float> > &multimodelweights) const; + + std::vector<std::vector<float> > + getWeights(size_t numWeights, bool normalize) const; + + std::vector<float> + normalizeWeights(std::vector<float> &weights) const; + + void + CacheForCleanup(TargetPhraseCollection::shared_ptr tpc); + + void + CleanUpAfterSentenceProcessing(const InputType &source); + + virtual void + CleanUpComponentModels(const InputType &source); + #ifdef WITH_DLIB virtual std::vector<float> MinimizePerplexity(std::vector<std::pair<std::string, std::string> > &phrase_pair_vector); std::vector<float> Optimize(OptimizationObjective * ObjectiveFunction, size_t numModels); #endif + // functions below required by base class - virtual const TargetPhraseCollection* GetTargetPhraseCollectionLEGACY(const Phrase& src) const; - virtual void InitializeForInput(ttasksptr const& ttask) { - /* Don't do anything source specific here as this object is shared between threads.*/ + virtual TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionLEGACY(const Phrase& src) const; + + virtual void + InitializeForInput(ttasksptr const& ttask) { + // Don't do anything source specific here as this object is shared + // between threads. } - ChartRuleLookupManager *CreateRuleLookupManager(const ChartParser &, const ChartCellCollectionBase&, std::size_t); - void SetParameter(const std::string& key, const std::string& value); - const std::vector<float>* GetTemporaryMultiModelWeightsVector() const; - void SetTemporaryMultiModelWeightsVector(std::vector<float> weights); + ChartRuleLookupManager* + CreateRuleLookupManager(const ChartParser &, const ChartCellCollectionBase&, + std::size_t); + + void + SetParameter(const std::string& key, const std::string& value); + + const std::vector<float>* + GetTemporaryMultiModelWeightsVector() const; + + void + SetTemporaryMultiModelWeightsVector(std::vector<float> weights); protected: std::string m_mode; @@ -101,7 +132,7 @@ protected: size_t m_numModels; std::vector<float> m_multimodelweights; - typedef std::vector<TargetPhraseCollection*> PhraseCache; + typedef std::vector<TargetPhraseCollection::shared_ptr> PhraseCache; #ifdef WITH_THREADS boost::shared_mutex m_lock_cache; typedef std::map<boost::thread::id, PhraseCache> SentenceCache; @@ -147,7 +178,7 @@ class CrossEntropy: public OptimizationObjective public: CrossEntropy ( - std::vector<multiModelStatisticsOptimization*> &optimizerStats, + std::vector<multiModelStatsOptimization*> &optimizerStats, PhraseDictionaryMultiModel * model, size_t iFeature ) { @@ -159,7 +190,7 @@ public: double operator() ( const dlib::matrix<double,0,1>& arg) const; protected: - std::vector<multiModelStatisticsOptimization*> m_optimizerStats; + std::vector<multiModelStatsOptimization*> m_optimizerStats; PhraseDictionaryMultiModel * m_model; size_t m_iFeature; }; diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp index 773e027cc..bd2552d9d 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp @@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ #include "util/exception.hh" #include "util/tokenize.hh" +#include "util/string_stream.hh" #include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h" using namespace std; @@ -56,7 +57,7 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons else if (m_mode == "interpolate") m_combineFunction = LinearInterpolationFromCounts; else { - ostringstream msg; + util::StringStream msg; msg << "combination mode unknown: " << m_mode; throw runtime_error(msg.str()); } @@ -119,7 +120,7 @@ void PhraseDictionaryMultiModelCounts::Load() } -const TargetPhraseCollection *PhraseDictionaryMultiModelCounts::GetTargetPhraseCollectionLEGACY(const Phrase& src) const +TargetPhraseCollection::shared_ptr PhraseDictionaryMultiModelCounts::GetTargetPhraseCollectionLEGACY(const Phrase& src) const { vector<vector<float> > multimodelweights; bool normalize; @@ -129,11 +130,12 @@ const TargetPhraseCollection *PhraseDictionaryMultiModelCounts::GetTargetPhraseC //source phrase frequency is shared among all phrase pairs vector<float> fs(m_numModels); - map<string,multiModelCountsStatistics*>* allStats = new(map<string,multiModelCountsStatistics*>); + map<string,multiModelCountsStats*>* allStats = new(map<string,multiModelCountsStats*>); - CollectSufficientStatistics(src, fs, allStats); + CollectSufficientStats(src, fs, allStats); - TargetPhraseCollection *ret = CreateTargetPhraseCollectionCounts(src, fs, allStats, multimodelweights); + TargetPhraseCollection::shared_ptr ret + = CreateTargetPhraseCollectionCounts(src, fs, allStats, multimodelweights); ret->NthElement(m_tableLimit); // sort the phrases for pruning later const_cast<PhraseDictionaryMultiModelCounts*>(this)->CacheForCleanup(ret); @@ -141,16 +143,17 @@ const TargetPhraseCollection *PhraseDictionaryMultiModelCounts::GetTargetPhraseC } -void PhraseDictionaryMultiModelCounts::CollectSufficientStatistics(const Phrase& src, vector<float> &fs, map<string,multiModelCountsStatistics*>* allStats) const +void PhraseDictionaryMultiModelCounts::CollectSufficientStats(const Phrase& src, vector<float> &fs, map<string,multiModelCountsStats*>* allStats) const //fill fs and allStats with statistics from models { for(size_t i = 0; i < m_numModels; ++i) { const PhraseDictionary &pd = *m_pd[i]; - TargetPhraseCollection *ret_raw = (TargetPhraseCollection*) pd.GetTargetPhraseCollectionLEGACY( src); + TargetPhraseCollection::shared_ptr ret_raw + = pd.GetTargetPhraseCollectionLEGACY(src); if (ret_raw != NULL) { - TargetPhraseCollection::iterator iterTargetPhrase; + TargetPhraseCollection::const_iterator iterTargetPhrase; for (iterTargetPhrase = ret_raw->begin(); iterTargetPhrase != ret_raw->end(); ++iterTargetPhrase) { const TargetPhrase * targetPhrase = *iterTargetPhrase; @@ -159,7 +162,7 @@ void PhraseDictionaryMultiModelCounts::CollectSufficientStatistics(const Phrase& string targetString = targetPhrase->GetStringRep(m_output); if (allStats->find(targetString) == allStats->end()) { - multiModelCountsStatistics * statistics = new multiModelCountsStatistics; + multiModelCountsStats * statistics = new multiModelCountsStats; statistics->targetPhrase = new TargetPhrase(*targetPhrase); //make a copy so that we don't overwrite the original phrase table info //correct future cost estimates and total score @@ -177,7 +180,7 @@ void PhraseDictionaryMultiModelCounts::CollectSufficientStatistics(const Phrase& (*allStats)[targetString] = statistics; } - multiModelCountsStatistics * statistics = (*allStats)[targetString]; + multiModelCountsStats * statistics = (*allStats)[targetString]; statistics->fst[i] = UntransformScore(raw_scores[0]); statistics->ft[i] = UntransformScore(raw_scores[1]); @@ -188,8 +191,8 @@ void PhraseDictionaryMultiModelCounts::CollectSufficientStatistics(const Phrase& } // get target phrase frequency for models which have not seen the phrase pair - for ( map< string, multiModelCountsStatistics*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { - multiModelCountsStatistics * statistics = iter->second; + for ( map< string, multiModelCountsStats*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { + multiModelCountsStats * statistics = iter->second; for (size_t i = 0; i < m_numModels; ++i) { if (!statistics->ft[i]) { @@ -199,12 +202,14 @@ void PhraseDictionaryMultiModelCounts::CollectSufficientStatistics(const Phrase& } } -TargetPhraseCollection* PhraseDictionaryMultiModelCounts::CreateTargetPhraseCollectionCounts(const Phrase &src, vector<float> &fs, map<string,multiModelCountsStatistics*>* allStats, vector<vector<float> > &multimodelweights) const +TargetPhraseCollection::shared_ptr +PhraseDictionaryMultiModelCounts:: +CreateTargetPhraseCollectionCounts(const Phrase &src, vector<float> &fs, map<string,multiModelCountsStats*>* allStats, vector<vector<float> > &multimodelweights) const { - TargetPhraseCollection *ret = new TargetPhraseCollection(); - for ( map< string, multiModelCountsStatistics*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { + TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection); + for ( map< string, multiModelCountsStats*>::const_iterator iter = allStats->begin(); iter != allStats->end(); ++iter ) { - multiModelCountsStatistics * statistics = iter->second; + multiModelCountsStats * statistics = iter->second; if (statistics->targetPhrase->GetAlignTerm().GetSize() == 0) { UTIL_THROW(util::Exception, " alignment information empty\ncount-tables need to include alignment information for computation of lexical weights.\nUse --phrase-word-alignment during training; for on-disk tables, also set -alignment-info when creating on-disk tables."); @@ -247,7 +252,7 @@ float PhraseDictionaryMultiModelCounts::GetTargetCount(const Phrase &target, siz { const PhraseDictionary &pd = *m_inverse_pd[modelIndex]; - const TargetPhraseCollection *ret_raw = pd.GetTargetPhraseCollectionLEGACY(target); + TargetPhraseCollection::shared_ptr ret_raw = pd.GetTargetPhraseCollectionLEGACY(target); // in inverse mode, we want the first score of the first phrase pair (note: if we were to work with truly symmetric models, it would be the third score) if (ret_raw && ret_raw->GetSize() > 0) { @@ -319,7 +324,7 @@ double PhraseDictionaryMultiModelCounts::ComputeWeightedLexicalTranslation( cons } -lexicalCache PhraseDictionaryMultiModelCounts::CacheLexicalStatistics( const Phrase &phraseS, const Phrase &phraseT, AlignVector &alignment, const vector<lexicalTable*> &tables, bool is_input ) +lexicalCache PhraseDictionaryMultiModelCounts::CacheLexicalStats( const Phrase &phraseS, const Phrase &phraseT, AlignVector &alignment, const vector<lexicalTable*> &tables, bool is_input ) { //do all the necessary lexical table lookups and get counts, but don't apply weights yet @@ -473,7 +478,7 @@ vector<float> PhraseDictionaryMultiModelCounts::MinimizePerplexity(vector<pair<s phrase_pair_map[*iter] += 1; } - vector<multiModelCountsStatisticsOptimization*> optimizerStats; + vector<multiModelCountsStatsOptimization*> optimizerStats; for ( map<pair<string, string>, size_t>::iterator iter = phrase_pair_map.begin(); iter != phrase_pair_map.end(); ++iter ) { @@ -482,12 +487,12 @@ vector<float> PhraseDictionaryMultiModelCounts::MinimizePerplexity(vector<pair<s string target_string = phrase_pair.second; vector<float> fs(m_numModels); - map<string,multiModelCountsStatistics*>* allStats = new(map<string,multiModelCountsStatistics*>); + map<string,multiModelCountsStats*>* allStats = new(map<string,multiModelCountsStats*>); Phrase sourcePhrase(0); sourcePhrase.CreateFromString(Input, m_input, source_string, NULL); - CollectSufficientStatistics(sourcePhrase, fs, allStats); //optimization potential: only call this once per source phrase + CollectSufficientStats(sourcePhrase, fs, allStats); //optimization potential: only call this once per source phrase //phrase pair not found; leave cache empty if (allStats->find(target_string) == allStats->end()) { @@ -496,19 +501,19 @@ vector<float> PhraseDictionaryMultiModelCounts::MinimizePerplexity(vector<pair<s continue; } - multiModelCountsStatisticsOptimization * targetStatistics = new multiModelCountsStatisticsOptimization(); - targetStatistics->targetPhrase = new TargetPhrase(*(*allStats)[target_string]->targetPhrase); - targetStatistics->fs = fs; - targetStatistics->fst = (*allStats)[target_string]->fst; - targetStatistics->ft = (*allStats)[target_string]->ft; - targetStatistics->f = iter->second; + multiModelCountsStatsOptimization * targetStats = new multiModelCountsStatsOptimization(); + targetStats->targetPhrase = new TargetPhrase(*(*allStats)[target_string]->targetPhrase); + targetStats->fs = fs; + targetStats->fst = (*allStats)[target_string]->fst; + targetStats->ft = (*allStats)[target_string]->ft; + targetStats->f = iter->second; try { - pair<vector< set<size_t> >, vector< set<size_t> > > alignment = GetAlignmentsForLexWeights(sourcePhrase, static_cast<const Phrase&>(*targetStatistics->targetPhrase), targetStatistics->targetPhrase->GetAlignTerm()); - targetStatistics->lexCachee2f = CacheLexicalStatistics(static_cast<const Phrase&>(*targetStatistics->targetPhrase), sourcePhrase, alignment.second, m_lexTable_e2f, false ); - targetStatistics->lexCachef2e = CacheLexicalStatistics(sourcePhrase, static_cast<const Phrase&>(*targetStatistics->targetPhrase), alignment.first, m_lexTable_f2e, true ); + pair<vector< set<size_t> >, vector< set<size_t> > > alignment = GetAlignmentsForLexWeights(sourcePhrase, static_cast<const Phrase&>(*targetStats->targetPhrase), targetStats->targetPhrase->GetAlignTerm()); + targetStats->lexCachee2f = CacheLexicalStats(static_cast<const Phrase&>(*targetStats->targetPhrase), sourcePhrase, alignment.second, m_lexTable_e2f, false ); + targetStats->lexCachef2e = CacheLexicalStats(sourcePhrase, static_cast<const Phrase&>(*targetStats->targetPhrase), alignment.first, m_lexTable_f2e, true ); - optimizerStats.push_back(targetStatistics); + optimizerStats.push_back(targetStats); } catch (AlignmentException& e) {} RemoveAllInMap(*allStats); @@ -560,8 +565,8 @@ double CrossEntropyCounts::operator() ( const dlib::matrix<double,0,1>& arg) con weight_vector = m_model->normalizeWeights(weight_vector); } - for ( std::vector<multiModelCountsStatisticsOptimization*>::const_iterator iter = m_optimizerStats.begin(); iter != m_optimizerStats.end(); ++iter ) { - multiModelCountsStatisticsOptimization* statistics = *iter; + for ( std::vector<multiModelCountsStatsOptimization*>::const_iterator iter = m_optimizerStats.begin(); iter != m_optimizerStats.end(); ++iter ) { + multiModelCountsStatsOptimization* statistics = *iter; size_t f = statistics->f; double score; diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.h b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.h index 7e4f32f30..5f59d826b 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.h +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.h @@ -37,11 +37,11 @@ typedef boost::unordered_map<Word, lexicalMap > lexicalMapJoint; typedef std::pair<std::vector<float>, std::vector<float> > lexicalPair; typedef std::vector<std::vector<lexicalPair> > lexicalCache; -struct multiModelCountsStatistics : multiModelStatistics { +struct multiModelCountsStats : multiModelStats { std::vector<float> fst, ft; }; -struct multiModelCountsStatisticsOptimization: multiModelCountsStatistics { +struct multiModelCountsStatsOptimization: multiModelCountsStats { std::vector<float> fs; lexicalCache lexCachee2f, lexCachef2e; size_t f; @@ -80,18 +80,18 @@ public: PhraseDictionaryMultiModelCounts(const std::string &line); ~PhraseDictionaryMultiModelCounts(); void Load(); - TargetPhraseCollection* CreateTargetPhraseCollectionCounts(const Phrase &src, std::vector<float> &fs, std::map<std::string,multiModelCountsStatistics*>* allStats, std::vector<std::vector<float> > &multimodelweights) const; - void CollectSufficientStatistics(const Phrase &src, std::vector<float> &fs, std::map<std::string,multiModelCountsStatistics*>* allStats) const; + TargetPhraseCollection::shared_ptr CreateTargetPhraseCollectionCounts(const Phrase &src, std::vector<float> &fs, std::map<std::string,multiModelCountsStats*>* allStats, std::vector<std::vector<float> > &multimodelweights) const; + void CollectSufficientStats(const Phrase &src, std::vector<float> &fs, std::map<std::string,multiModelCountsStats*>* allStats) const; float GetTargetCount(const Phrase& target, size_t modelIndex) const; double GetLexicalProbability( Word &inner, Word &outer, const std::vector<lexicalTable*> &tables, std::vector<float> &multimodelweights ) const; double ComputeWeightedLexicalTranslation( const Phrase &phraseS, const Phrase &phraseT, AlignVector &alignment, const std::vector<lexicalTable*> &tables, std::vector<float> &multimodelweights, bool is_input ) const; double ComputeWeightedLexicalTranslationFromCache( std::vector<std::vector<std::pair<std::vector<float>, std::vector<float> > > > &cache, std::vector<float> &weights ) const; std::pair<PhraseDictionaryMultiModelCounts::AlignVector,PhraseDictionaryMultiModelCounts::AlignVector> GetAlignmentsForLexWeights(const Phrase &phraseS, const Phrase &phraseT, const AlignmentInfo &alignment) const; - std::vector<std::vector<std::pair<std::vector<float>, std::vector<float> > > > CacheLexicalStatistics( const Phrase &phraseS, const Phrase &phraseT, AlignVector &alignment, const std::vector<lexicalTable*> &tables, bool is_input ); + std::vector<std::vector<std::pair<std::vector<float>, std::vector<float> > > > CacheLexicalStats( const Phrase &phraseS, const Phrase &phraseT, AlignVector &alignment, const std::vector<lexicalTable*> &tables, bool is_input ); void FillLexicalCountsJoint(Word &wordS, Word &wordT, std::vector<float> &count, const std::vector<lexicalTable*> &tables) const; void FillLexicalCountsMarginal(Word &wordS, std::vector<float> &count, const std::vector<lexicalTable*> &tables) const; void LoadLexicalTable( std::string &fileName, lexicalTable* ltable); - const TargetPhraseCollection* GetTargetPhraseCollectionLEGACY(const Phrase& src) const; + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionLEGACY(const Phrase& src) const; #ifdef WITH_DLIB std::vector<float> MinimizePerplexity(std::vector<std::pair<std::string, std::string> > &phrase_pair_vector); #endif @@ -117,7 +117,7 @@ class CrossEntropyCounts: public OptimizationObjective public: CrossEntropyCounts ( - std::vector<multiModelCountsStatisticsOptimization*> &optimizerStats, + std::vector<multiModelCountsStatsOptimization*> &optimizerStats, PhraseDictionaryMultiModelCounts * model, size_t iFeature ) { @@ -129,7 +129,7 @@ public: double operator() ( const dlib::matrix<double,0,1>& arg) const; private: - std::vector<multiModelCountsStatisticsOptimization*> m_optimizerStats; + std::vector<multiModelCountsStatsOptimization*> m_optimizerStats; PhraseDictionaryMultiModelCounts * m_model; size_t m_iFeature; }; diff --git a/moses/TranslationModel/PhraseDictionaryNodeMemory.cpp b/moses/TranslationModel/PhraseDictionaryNodeMemory.cpp index 84639a737..0c562d4c1 100644 --- a/moses/TranslationModel/PhraseDictionaryNodeMemory.cpp +++ b/moses/TranslationModel/PhraseDictionaryNodeMemory.cpp @@ -39,7 +39,7 @@ void PhraseDictionaryNodeMemory::Prune(size_t tableLimit) } // prune TargetPhraseCollection in this node - m_targetPhraseCollection.Prune(true, tableLimit); + m_targetPhraseCollection->Prune(true, tableLimit); } void PhraseDictionaryNodeMemory::Sort(size_t tableLimit) @@ -53,10 +53,11 @@ void PhraseDictionaryNodeMemory::Sort(size_t tableLimit) } // prune TargetPhraseCollection in this node - m_targetPhraseCollection.Sort(true, tableLimit); + m_targetPhraseCollection->Sort(true, tableLimit); } -PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetOrCreateChild(const Word &sourceTerm) +PhraseDictionaryNodeMemory* +PhraseDictionaryNodeMemory::GetOrCreateChild(const Word &sourceTerm) { return &m_sourceTermMap[sourceTerm]; } @@ -118,7 +119,7 @@ void PhraseDictionaryNodeMemory::Remove() { m_sourceTermMap.clear(); m_nonTermMap.clear(); - m_targetPhraseCollection.Remove(); + m_targetPhraseCollection->Remove(); } std::ostream& operator<<(std::ostream &out, const PhraseDictionaryNodeMemory &node) diff --git a/moses/TranslationModel/PhraseDictionaryNodeMemory.h b/moses/TranslationModel/PhraseDictionaryNodeMemory.h index 950838ae3..8608b90a1 100644 --- a/moses/TranslationModel/PhraseDictionaryNodeMemory.h +++ b/moses/TranslationModel/PhraseDictionaryNodeMemory.h @@ -130,11 +130,12 @@ private: TerminalMap m_sourceTermMap; NonTerminalMap m_nonTermMap; - TargetPhraseCollection m_targetPhraseCollection; + TargetPhraseCollection::shared_ptr m_targetPhraseCollection; public: - PhraseDictionaryNodeMemory() {} + PhraseDictionaryNodeMemory() + : m_targetPhraseCollection(new TargetPhraseCollection) { } bool IsLeaf() const { return m_sourceTermMap.empty() && m_nonTermMap.empty(); @@ -152,10 +153,12 @@ public: const PhraseDictionaryNodeMemory *GetChild(const Word &sourceNonTerm, const Word &targetNonTerm) const; #endif - const TargetPhraseCollection &GetTargetPhraseCollection() const { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() const { return m_targetPhraseCollection; } - TargetPhraseCollection &GetTargetPhraseCollection() { + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection() { return m_targetPhraseCollection; } diff --git a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp index 03b69d0ad..13ca1d1d1 100644 --- a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp +++ b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp @@ -54,7 +54,9 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const Input } } -void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &inputPath) const +void +PhraseDictionaryTransliteration:: +GetTargetPhraseCollection(InputPath &inputPath) const { const Phrase &sourcePhrase = inputPath.GetPhrase(); size_t hash = hash_value(sourcePhrase); @@ -66,7 +68,7 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &input if (iter != cache.end()) { // already in cache - const TargetPhraseCollection *tpColl = iter->second.first; + TargetPhraseCollection::shared_ptr tpColl = iter->second.first; inputPath.SetTargetPhrases(*this, tpColl, NULL); } else { // TRANSLITERATE @@ -89,17 +91,15 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &input int ret = system(cmd.c_str()); UTIL_THROW_IF2(ret != 0, "Transliteration script error"); - TargetPhraseCollection *tpColl = new TargetPhraseCollection(); - vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir.path()); + TargetPhraseCollection::shared_ptr tpColl(new TargetPhraseCollection); + vector<TargetPhrase*> targetPhrases + = CreateTargetPhrases(sourcePhrase, outDir.path()); vector<TargetPhrase*>::const_iterator iter; for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) { TargetPhrase *tp = *iter; tpColl->Add(tp); } - - std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock()); - cache[hash] = value; - + cache[hash] = CacheCollEntry(tpColl, clock()); inputPath.SetTargetPhrases(*this, tpColl, NULL); } } diff --git a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp index ca1638708..fbeb3abe6 100644 --- a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp +++ b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp @@ -15,6 +15,7 @@ #include "moses/PDTAimp.h" #include "moses/TranslationTask.h" #include "util/exception.hh" +#include "util/string_stream.hh" using namespace std; @@ -52,7 +53,7 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(ttasksptr const& ttask) vector<float> weight = staticData.GetWeights(this); if(m_numScoreComponents!=weight.size()) { - std::stringstream strme; + util::StringStream strme; UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size() << " " << m_numScoreComponents); } @@ -73,11 +74,10 @@ void PhraseDictionaryTreeAdaptor::CleanUpAfterSentenceProcessing(InputType const obj.CleanUp(); } -TargetPhraseCollection const* +TargetPhraseCollection::shared_ptr PhraseDictionaryTreeAdaptor::GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const { - const TargetPhraseCollection *ret = GetImplementation().GetTargetPhraseCollection(src); - return ret; + return GetImplementation().GetTargetPhraseCollection(src); } void PhraseDictionaryTreeAdaptor::EnableCache() @@ -106,16 +106,17 @@ const PDTAimp& PhraseDictionaryTreeAdaptor::GetImplementation() const } // legacy -const TargetPhraseCollectionWithSourcePhrase* -PhraseDictionaryTreeAdaptor::GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const &range) const +TargetPhraseCollectionWithSourcePhrase::shared_ptr +PhraseDictionaryTreeAdaptor:: +GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const &range) const { + TargetPhraseCollectionWithSourcePhrase::shared_ptr ret; if(GetImplementation().m_rangeCache.empty()) { - const TargetPhraseCollectionWithSourcePhrase *tpColl = GetImplementation().GetTargetPhraseCollection(src.GetSubString(range)); - return tpColl; + ret = GetImplementation().GetTargetPhraseCollection(src.GetSubString(range)); } else { - const TargetPhraseCollectionWithSourcePhrase *tpColl = GetImplementation().m_rangeCache[range.GetStartPos()][range.GetEndPos()]; - return tpColl; + ret = GetImplementation().m_rangeCache[range.GetStartPos()][range.GetEndPos()]; } + return ret; } } diff --git a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.h b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.h index 0ed8ed612..020ce117e 100644 --- a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.h +++ b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.h @@ -59,7 +59,8 @@ public: // get translation candidates for a given source phrase // returns null pointer if nothing found - TargetPhraseCollection const* GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const; + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionNonCacheLEGACY(Phrase const &src) const; void InitializeForInput(ttasksptr const& ttask); void CleanUpAfterSentenceProcessing(InputType const& source); @@ -73,7 +74,9 @@ public: } // legacy - const TargetPhraseCollectionWithSourcePhrase *GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const & srcRange) const; + TargetPhraseCollectionWithSourcePhrase::shared_ptr + GetTargetPhraseCollectionLEGACY(InputType const& src, + WordsRange const & srcRange) const; }; diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.cpp b/moses/TranslationModel/ProbingPT/ProbingPT.cpp index 19b7e8795..391391ab7 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp +++ b/moses/TranslationModel/ProbingPT/ProbingPT.cpp @@ -2,6 +2,7 @@ #include "ProbingPT.h" #include "moses/StaticData.h" #include "moses/FactorCollection.h" +#include "moses/TargetPhraseCollection.h" #include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" #include "quering.hh" @@ -79,11 +80,11 @@ void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQue continue; } - TargetPhraseCollection *tpColl = CreateTargetPhrase(sourcePhrase); + TargetPhraseCollection::shared_ptr tpColl = CreateTargetPhrase(sourcePhrase); // add target phrase to phrase-table cache size_t hash = hash_value(sourcePhrase); - std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock()); + std::pair<TargetPhraseCollection::shared_ptr , clock_t> value(tpColl, clock()); cache[hash] = value; inputPath.SetTargetPhrases(*this, tpColl, NULL); @@ -109,29 +110,28 @@ std::vector<uint64_t> ProbingPT::ConvertToProbingSourcePhrase(const Phrase &sour return ret; } -TargetPhraseCollection *ProbingPT::CreateTargetPhrase(const Phrase &sourcePhrase) const +TargetPhraseCollection::shared_ptr ProbingPT::CreateTargetPhrase(const Phrase &sourcePhrase) const { // create a target phrase from the 1st word of the source, prefix with 'ProbingPT:' assert(sourcePhrase.GetSize()); + TargetPhraseCollection::shared_ptr tpColl; bool ok; vector<uint64_t> probingSource = ConvertToProbingSourcePhrase(sourcePhrase, ok); if (!ok) { // source phrase contains a word unknown in the pt. // We know immediately there's no translation for it - return NULL; + return tpColl; } std::pair<bool, std::vector<target_text> > query_result; - TargetPhraseCollection *tpColl = NULL; - //Actual lookup query_result = m_engine->query(probingSource); if (query_result.first) { //m_engine->printTargetInfo(query_result.second); - tpColl = new TargetPhraseCollection(); + tpColl.reset(new TargetPhraseCollection()); const std::vector<target_text> &probingTargetPhrases = query_result.second; for (size_t i = 0; i < probingTargetPhrases.size(); ++i) { diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.h b/moses/TranslationModel/ProbingPT/ProbingPT.h index c6cc69c89..48f4441f9 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.h +++ b/moses/TranslationModel/ProbingPT/ProbingPT.h @@ -46,7 +46,7 @@ protected: typedef boost::bimap<const Factor *, unsigned int> TargetVocabMap; mutable TargetVocabMap m_vocabMap; - TargetPhraseCollection *CreateTargetPhrase(const Phrase &sourcePhrase) const; + TargetPhraseCollection::shared_ptr CreateTargetPhrase(const Phrase &sourcePhrase) const; TargetPhrase *CreateTargetPhrase(const Phrase &sourcePhrase, const target_text &probingTargetPhrase) const; const Factor *GetTargetFactor(uint64_t probingId) const; uint64_t GetSourceProbingId(const Factor *factor) const; diff --git a/moses/TranslationModel/RuleTable/Loader.h b/moses/TranslationModel/RuleTable/Loader.h index 48390e37e..b3aed9dc0 100644 --- a/moses/TranslationModel/RuleTable/Loader.h +++ b/moses/TranslationModel/RuleTable/Loader.h @@ -49,12 +49,13 @@ protected: // Provide access to RuleTableTrie's private // GetOrCreateTargetPhraseCollection function. - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - RuleTableTrie &ruleTable - , const Phrase &source - , const TargetPhrase &target - , const Word *sourceLHS) { - return ruleTable.GetOrCreateTargetPhraseCollection(source, target, sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(RuleTableTrie &ruleTable, + const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) { + return ruleTable.GetOrCreateTargetPhraseCollection(source, target, + sourceLHS); } }; diff --git a/moses/TranslationModel/RuleTable/LoaderCompact.cpp b/moses/TranslationModel/RuleTable/LoaderCompact.cpp index c947dfdc2..945a49bfc 100644 --- a/moses/TranslationModel/RuleTable/LoaderCompact.cpp +++ b/moses/TranslationModel/RuleTable/LoaderCompact.cpp @@ -224,9 +224,10 @@ bool RuleTableLoaderCompact::LoadRuleSection( targetPhrase->EvaluateInIsolation(sourcePhrase, ruleTable.GetFeaturesToApply()); // Insert rule into table. - TargetPhraseCollection &coll = GetOrCreateTargetPhraseCollection( - ruleTable, sourcePhrase, *targetPhrase, &sourceLHS); - coll.Add(targetPhrase); + TargetPhraseCollection::shared_ptr coll; + coll = GetOrCreateTargetPhraseCollection(ruleTable, sourcePhrase, + *targetPhrase, &sourceLHS); + coll->Add(targetPhrase); } return true; diff --git a/moses/TranslationModel/RuleTable/LoaderStandard.cpp b/moses/TranslationModel/RuleTable/LoaderStandard.cpp index f9e6ac6fd..2c2c3a8be 100644 --- a/moses/TranslationModel/RuleTable/LoaderStandard.cpp +++ b/moses/TranslationModel/RuleTable/LoaderStandard.cpp @@ -126,14 +126,14 @@ void ReformatHieroRule(const string &lineOrig, string &out) ReformatHieroRule(1, targetPhraseString, ntAlign); ReformateHieroScore(scoreString); - stringstream align; + util::StringStream align; map<size_t, pair<size_t, size_t> >::const_iterator iterAlign; for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) { const pair<size_t, size_t> &alignPoint = iterAlign->second; align << alignPoint.first << "-" << alignPoint.second << " "; } - stringstream ret; + util::StringStream ret; ret << sourcePhraseString << " ||| " << targetPhraseString << " ||| " << scoreString << " ||| " @@ -242,8 +242,10 @@ bool RuleTableLoaderStandard::Load(FormatType format targetPhrase->GetScoreBreakdown().Assign(&ruleTable, scoreVector); targetPhrase->EvaluateInIsolation(sourcePhrase, ruleTable.GetFeaturesToApply()); - TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection(ruleTable, sourcePhrase, *targetPhrase, sourceLHS); - phraseColl.Add(targetPhrase); + TargetPhraseCollection::shared_ptr phraseColl + = GetOrCreateTargetPhraseCollection(ruleTable, sourcePhrase, + *targetPhrase, sourceLHS); + phraseColl->Add(targetPhrase); // not implemented correctly in memory pt. just delete it for now delete sourceLHS; diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp index 5f7ddf85d..0d6321f2b 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp @@ -282,8 +282,10 @@ void PhraseDictionaryFuzzyMatch::InitializeForInput(ttasksptr const& ttask) targetPhrase->GetScoreBreakdown().Assign(this, scoreVector); targetPhrase->EvaluateInIsolation(sourcePhrase, GetFeaturesToApply()); - TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection(rootNode, sourcePhrase, *targetPhrase, sourceLHS); - phraseColl.Add(targetPhrase); + TargetPhraseCollection::shared_ptr phraseColl + = GetOrCreateTargetPhraseCollection(rootNode, sourcePhrase, + *targetPhrase, sourceLHS); + phraseColl->Add(targetPhrase); count++; @@ -301,10 +303,12 @@ void PhraseDictionaryFuzzyMatch::InitializeForInput(ttasksptr const& ttask) //removedirectoryrecursively(dirName); } -TargetPhraseCollection &PhraseDictionaryFuzzyMatch::GetOrCreateTargetPhraseCollection(PhraseDictionaryNodeMemory &rootNode - , const Phrase &source - , const TargetPhrase &target - , const Word *sourceLHS) +TargetPhraseCollection::shared_ptr +PhraseDictionaryFuzzyMatch:: +GetOrCreateTargetPhraseCollection(PhraseDictionaryNodeMemory &rootNode + , const Phrase &source + , const TargetPhrase &target + , const Word *sourceLHS) { PhraseDictionaryNodeMemory &currNode = GetOrCreateNode(rootNode, source, target, sourceLHS); return currNode.GetTargetPhraseCollection(); diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h index 8751b7956..5c710021a 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h @@ -1,3 +1,4 @@ +// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*- /*********************************************************************** Moses - statistical machine translation system Copyright (C) 2006-2011 University of Edinburgh @@ -59,10 +60,11 @@ public: TO_STRING(); protected: - TargetPhraseCollection &GetOrCreateTargetPhraseCollection(PhraseDictionaryNodeMemory &rootNode - , const Phrase &source - , const TargetPhrase &target - , const Word *sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(PhraseDictionaryNodeMemory &rootNode + , const Phrase &source + , const TargetPhrase &target + , const Word *sourceLHS); PhraseDictionaryNodeMemory &GetOrCreateNode(PhraseDictionaryNodeMemory &rootNode , const Phrase &source diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp index e99385d82..3eba6e5de 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp @@ -149,26 +149,26 @@ void PhraseDictionaryOnDisk::GetTargetPhraseCollectionBatch(InputPath &inputPath lastWord.OnlyTheseFactors(m_inputFactors); OnDiskPt::Word *lastWordOnDisk = wrapper.ConvertFromMoses(m_input, lastWord); + TargetPhraseCollection::shared_ptr tpc; if (lastWordOnDisk == NULL) { // OOV according to this phrase table. Not possible to extend - inputPath.SetTargetPhrases(*this, NULL, NULL); + inputPath.SetTargetPhrases(*this, tpc, NULL); } else { - const OnDiskPt::PhraseNode *ptNode = prevPtNode->GetChild(*lastWordOnDisk, wrapper); - if (ptNode) { - const TargetPhraseCollection *targetPhrases = GetTargetPhraseCollection(ptNode); - inputPath.SetTargetPhrases(*this, targetPhrases, ptNode); - } else { - inputPath.SetTargetPhrases(*this, NULL, NULL); - } + OnDiskPt::PhraseNode const* ptNode; + ptNode = prevPtNode->GetChild(*lastWordOnDisk, wrapper); + if (ptNode) tpc = GetTargetPhraseCollection(ptNode); + inputPath.SetTargetPhrases(*this, tpc, ptNode); delete lastWordOnDisk; } } } -const TargetPhraseCollection *PhraseDictionaryOnDisk::GetTargetPhraseCollection(const OnDiskPt::PhraseNode *ptNode) const +TargetPhraseCollection::shared_ptr +PhraseDictionaryOnDisk:: +GetTargetPhraseCollection(const OnDiskPt::PhraseNode *ptNode) const { - const TargetPhraseCollection *ret; + TargetPhraseCollection::shared_ptr ret; CacheColl &cache = GetCache(); size_t hash = (size_t) ptNode->GetFilePos(); @@ -181,31 +181,34 @@ const TargetPhraseCollection *PhraseDictionaryOnDisk::GetTargetPhraseCollection( // not in cache, need to look up from phrase table ret = GetTargetPhraseCollectionNonCache(ptNode); - std::pair<const TargetPhraseCollection*, clock_t> value(ret, clock()); + std::pair<TargetPhraseCollection::shared_ptr , clock_t> value(ret, clock()); cache[hash] = value; } else { // in cache. just use it - std::pair<const TargetPhraseCollection*, clock_t> &value = iter->second; - value.second = clock(); - - ret = value.first; + iter->second.second = clock(); + ret = iter->second.first; } return ret; } -const TargetPhraseCollection *PhraseDictionaryOnDisk::GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const +TargetPhraseCollection::shared_ptr +PhraseDictionaryOnDisk:: +GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const { - OnDiskPt::OnDiskWrapper &wrapper = const_cast<OnDiskPt::OnDiskWrapper&>(GetImplementation()); + OnDiskPt::OnDiskWrapper& wrapper + = const_cast<OnDiskPt::OnDiskWrapper&>(GetImplementation()); vector<float> weightT = StaticData::Instance().GetWeights(this); OnDiskPt::Vocab &vocab = wrapper.GetVocab(); - const OnDiskPt::TargetPhraseCollection *targetPhrasesOnDisk = ptNode->GetTargetPhraseCollection(m_tableLimit, wrapper); - TargetPhraseCollection *targetPhrases - = targetPhrasesOnDisk->ConvertToMoses(m_input, m_output, *this, weightT, vocab, false); + OnDiskPt::TargetPhraseCollection::shared_ptr targetPhrasesOnDisk + = ptNode->GetTargetPhraseCollection(m_tableLimit, wrapper); + TargetPhraseCollection::shared_ptr targetPhrases + = targetPhrasesOnDisk->ConvertToMoses(m_input, m_output, *this, + weightT, vocab, false); - delete targetPhrasesOnDisk; + // delete targetPhrasesOnDisk; return targetPhrases; } diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h index 246690922..c3449e972 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h @@ -78,8 +78,11 @@ public: virtual void InitializeForInput(ttasksptr const& ttask); void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; - const TargetPhraseCollection *GetTargetPhraseCollection(const OnDiskPt::PhraseNode *ptNode) const; - const TargetPhraseCollection *GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const; + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollection(const OnDiskPt::PhraseNode *ptNode) const; + + TargetPhraseCollection::shared_ptr + GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const; void SetParameter(const std::string& key, const std::string& value); diff --git a/moses/TranslationModel/RuleTable/Trie.h b/moses/TranslationModel/RuleTable/Trie.h index 51cc92e4a..76cbc16e5 100644 --- a/moses/TranslationModel/RuleTable/Trie.h +++ b/moses/TranslationModel/RuleTable/Trie.h @@ -51,9 +51,10 @@ public: private: friend class RuleTableLoader; - virtual TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, - const Word *sourceLHS) = 0; + virtual TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) = 0; virtual void SortAndPrune() = 0; diff --git a/moses/TranslationModel/RuleTable/UTrie.cpp b/moses/TranslationModel/RuleTable/UTrie.cpp index d6ccb4c78..eee88a11b 100644 --- a/moses/TranslationModel/RuleTable/UTrie.cpp +++ b/moses/TranslationModel/RuleTable/UTrie.cpp @@ -38,8 +38,11 @@ namespace Moses { -TargetPhraseCollection &RuleTableUTrie::GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS) +TargetPhraseCollection::shared_ptr +RuleTableUTrie:: +GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS) { UTrieNode &currNode = GetOrCreateNode(source, target, sourceLHS); return currNode.GetOrCreateTargetPhraseCollection(target); diff --git a/moses/TranslationModel/RuleTable/UTrie.h b/moses/TranslationModel/RuleTable/UTrie.h index caab05b04..708bf866e 100644 --- a/moses/TranslationModel/RuleTable/UTrie.h +++ b/moses/TranslationModel/RuleTable/UTrie.h @@ -21,13 +21,13 @@ #include "Trie.h" #include "UTrieNode.h" +#include "moses/TargetPhraseCollection.h" namespace Moses { class Phrase; class TargetPhrase; -class TargetPhraseCollection; class Word; class ChartParser; @@ -57,8 +57,10 @@ public: const ChartCellCollectionBase &, std::size_t); private: - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const Phrase &source, + const TargetPhrase &target, + const Word *sourceLHS); UTrieNode &GetOrCreateNode(const Phrase &source, const TargetPhrase &target, const Word *sourceLHS); diff --git a/moses/TranslationModel/RuleTable/UTrieNode.cpp b/moses/TranslationModel/RuleTable/UTrieNode.cpp index 725f02c97..d85dc662e 100644 --- a/moses/TranslationModel/RuleTable/UTrieNode.cpp +++ b/moses/TranslationModel/RuleTable/UTrieNode.cpp @@ -49,7 +49,7 @@ void UTrieNode::Prune(size_t tableLimit) // Prune TargetPhraseCollections at this node. for (LabelMap::iterator p = m_labelMap.begin(); p != m_labelMap.end(); ++p) { - p->second.Prune(true, tableLimit); + p->second->Prune(true, tableLimit); } } @@ -66,7 +66,7 @@ void UTrieNode::Sort(size_t tableLimit) // Sort TargetPhraseCollections at this node. for (LabelMap::iterator p = m_labelMap.begin(); p != m_labelMap.end(); ++p) { - p->second.Sort(true, tableLimit); + p->second->Sort(true, tableLimit); } } @@ -89,8 +89,9 @@ UTrieNode *UTrieNode::GetOrCreateNonTerminalChild(const Word &targetNonTerm) return m_gapNode; } -TargetPhraseCollection &UTrieNode::GetOrCreateTargetPhraseCollection( - const TargetPhrase &target) +TargetPhraseCollection::shared_ptr +UTrieNode:: +GetOrCreateTargetPhraseCollection(const TargetPhrase &target) { const AlignmentInfo &alignmentInfo = target.GetAlignNonTerm(); const size_t rank = alignmentInfo.GetSize(); @@ -107,8 +108,9 @@ TargetPhraseCollection &UTrieNode::GetOrCreateTargetPhraseCollection( const Word &targetNonTerm = target.GetWord(targetNonTermIndex); vec.push_back(InsertLabel(i++, targetNonTerm)); } - - return m_labelMap[vec]; + TargetPhraseCollection::shared_ptr& ret = m_labelMap[vec]; + if (ret == NULL) ret.reset(new TargetPhraseCollection); + return ret; } } // namespace Moses diff --git a/moses/TranslationModel/RuleTable/UTrieNode.h b/moses/TranslationModel/RuleTable/UTrieNode.h index 436bcbea1..96ba09e85 100644 --- a/moses/TranslationModel/RuleTable/UTrieNode.h +++ b/moses/TranslationModel/RuleTable/UTrieNode.h @@ -51,10 +51,10 @@ public: TerminalEqualityPred> TerminalMap; typedef boost::unordered_map<std::vector<int>, - TargetPhraseCollection> LabelMap; + TargetPhraseCollection::shared_ptr> LabelMap; #else typedef std::map<Word, UTrieNode> TerminalMap; - typedef std::map<std::vector<int>, TargetPhraseCollection> LabelMap; + typedef std::map<std::vector<int>, TargetPhraseCollection::shared_ptr> LabelMap; #endif ~UTrieNode() { @@ -78,8 +78,8 @@ public: UTrieNode *GetOrCreateTerminalChild(const Word &sourceTerm); UTrieNode *GetOrCreateNonTerminalChild(const Word &targetNonTerm); - TargetPhraseCollection &GetOrCreateTargetPhraseCollection( - const TargetPhrase &); + TargetPhraseCollection::shared_ptr + GetOrCreateTargetPhraseCollection(const TargetPhrase &); bool IsLeaf() const { return m_terminalMap.empty() && m_gapNode == NULL; diff --git a/moses/TranslationModel/Scope3Parser/Parser.cpp b/moses/TranslationModel/Scope3Parser/Parser.cpp index c8c8c3e49..fe1561c6e 100644 --- a/moses/TranslationModel/Scope3Parser/Parser.cpp +++ b/moses/TranslationModel/Scope3Parser/Parser.cpp @@ -47,7 +47,8 @@ void Scope3Parser::GetChartRuleCollection( const size_t start = range.GetStartPos(); const size_t end = range.GetEndPos(); - std::vector<std::pair<const UTrieNode *, const VarSpanNode *> > &pairVec = m_ruleApplications[start][end-start+1]; + std::vector<std::pair<const UTrieNode *, const VarSpanNode *> > &pairVec + = m_ruleApplications[start][end-start+1]; MatchCallback matchCB(range, outColl); for (std::vector<std::pair<const UTrieNode *, const VarSpanNode *> >::const_iterator p = pairVec.begin(); p != pairVec.end(); ++p) { @@ -58,8 +59,8 @@ void Scope3Parser::GetChartRuleCollection( if (varSpanNode.m_rank == 0) { // Purely lexical rule. assert(labelMap.size() == 1); - const TargetPhraseCollection &tpc = labelMap.begin()->second; - matchCB.m_tpc = &tpc; + TargetPhraseCollection::shared_ptr tpc = labelMap.begin()->second; + matchCB.m_tpc = tpc; matchCB(m_emptyStackVec); } else { // Rule has at least one non-terminal. varSpanNode.CalculateRanges(start, end, m_ranges); @@ -70,7 +71,7 @@ void Scope3Parser::GetChartRuleCollection( UTrieNode::LabelMap::const_iterator p = labelMap.begin(); for (; p != labelMap.end(); ++p) { const std::vector<int> &labels = p->first; - const TargetPhraseCollection &tpc = p->second; + TargetPhraseCollection::shared_ptr tpc = p->second; assert(labels.size() == varSpanNode.m_rank); bool failCheck = false; for (size_t i = 0; i < varSpanNode.m_rank; ++i) { @@ -82,7 +83,7 @@ void Scope3Parser::GetChartRuleCollection( if (failCheck) { continue; } - matchCB.m_tpc = &tpc; + matchCB.m_tpc = tpc; searcher.Search(labels, matchCB); } } diff --git a/moses/TranslationModel/Scope3Parser/Parser.h b/moses/TranslationModel/Scope3Parser/Parser.h index 70b26b50d..e7f4a5d51 100644 --- a/moses/TranslationModel/Scope3Parser/Parser.h +++ b/moses/TranslationModel/Scope3Parser/Parser.h @@ -66,17 +66,16 @@ private: // Define a callback type for use by StackLatticeSearcher. struct MatchCallback { public: - MatchCallback(const WordsRange &range, - ChartParserCallback &out) - : m_range(range) - , m_out(out) - , m_tpc(NULL) {} + MatchCallback(const WordsRange &range, ChartParserCallback &out) + : m_range(range) , m_out(out) // , m_tpc(NULL) + { } + void operator()(const StackVec &stackVec) { m_out.Add(*m_tpc, stackVec, m_range); } const WordsRange &m_range; ChartParserCallback &m_out; - const TargetPhraseCollection *m_tpc; + TargetPhraseCollection::shared_ptr m_tpc; }; void Init(); diff --git a/moses/TranslationModel/SkeletonPT.cpp b/moses/TranslationModel/SkeletonPT.cpp index 8e2b1daa3..1b8e0cfa1 100644 --- a/moses/TranslationModel/SkeletonPT.cpp +++ b/moses/TranslationModel/SkeletonPT.cpp @@ -32,12 +32,13 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQu const Phrase &sourcePhrase = inputPath.GetPhrase(); TargetPhrase *tp = CreateTargetPhrase(sourcePhrase); - TargetPhraseCollection *tpColl = new TargetPhraseCollection(); + TargetPhraseCollection::shared_ptr tpColl(new TargetPhraseCollection); tpColl->Add(tp); // add target phrase to phrase-table cache size_t hash = hash_value(sourcePhrase); - std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock()); + std::pair<TargetPhraseCollection::shared_ptr, clock_t> + value(tpColl, clock()); cache[hash] = value; inputPath.SetTargetPhrases(*this, tpColl, NULL); diff --git a/moses/TranslationModel/UG/TargetPhraseCollectionCache.cc b/moses/TranslationModel/UG/TargetPhraseCollectionCache.cc index 54f7ffa5a..89927b21c 100644 --- a/moses/TranslationModel/UG/TargetPhraseCollectionCache.cc +++ b/moses/TranslationModel/UG/TargetPhraseCollectionCache.cc @@ -4,183 +4,103 @@ namespace Moses { using std::vector; -#if defined(timespec) - bool operator<(timespec const& a, timespec const& b) + TPCollCache:: + TPCollCache(size_t capacity) { - if (a.tv_sec != b.tv_sec) return a.tv_sec < b.tv_sec; - return (a.tv_nsec < b.tv_nsec); + m_qfirst = m_qlast = m_cache.end(); + m_capacity = capacity; + UTIL_THROW_IF2(m_capacity <= 2, "Cache capacity must be > 1!"); } - bool operator>=(timespec const& a, timespec const& b) + SPTR<TPCollWrapper> + TPCollCache:: + get(uint64_t key, size_t revision) { - if (a.tv_sec != b.tv_sec) return a.tv_sec > b.tv_sec; - return (a.tv_nsec >= b.tv_nsec); - } -#endif - - bool operator<(timeval const& a, timeval const& b) - { - if (a.tv_sec != b.tv_sec) return a.tv_sec < b.tv_sec; - return (a.tv_usec < b.tv_usec); - } + using namespace boost; + unique_lock<shared_mutex> lock(m_lock); - bool operator>=(timeval const& a, timeval const& b) - { - if (a.tv_sec != b.tv_sec) return a.tv_sec > b.tv_sec; - return (a.tv_usec >= b.tv_usec); - } + size_t ctr=0; +#if 0 + std::cerr << "BEFORE" << std::endl; + for (cache_t::iterator m = m_qfirst; m != m_cache.end(); m = m->second->next) + { + std::cerr << ++ctr << "/" << m_cache.size() << " " + << (m->second->key == key ? "*" : " ") + << m->second->key << " " + << m->second.use_count(); + if (m->second->prev != m_cache.end()) + std::cerr << " => " << m->second->prev->second->key; + std::cerr << std::endl; + } + std::cerr << "\n" << std::endl; +#endif - void - bubble_up(std::vector<TPCollWrapper*>& v, size_t k) - { - if (k >= v.size()) return; - for (;k && (v[k]->tstamp < v[k/2]->tstamp); k /=2) + std::pair<uint64_t, SPTR<TPCollWrapper> > e(key, SPTR<TPCollWrapper>()); + std::pair<cache_t::iterator, bool> foo = m_cache.insert(e); + SPTR<TPCollWrapper>& ret = foo.first->second; + if (ret && m_cache.size() > 1 && m_qlast != foo.first) { - std::swap(v[k],v[k/2]); - std::swap(v[k]->idx,v[k/2]->idx); + if (m_qfirst == foo.first) m_qfirst = ret->next; + else ret->prev->second->next = ret->next; + if (m_qlast != foo.first) + ret->next->second->prev = ret->prev; } - } - - void - bubble_down(std::vector<TPCollWrapper*>& v, size_t k) - { - for (size_t j = 2*(k+1); j <= v.size(); j = 2*((k=j)+1)) + if (!ret || ret->revision != revision) { - if (j == v.size() || (v[j-1]->tstamp < v[j]->tstamp)) --j; - if (v[j]->tstamp >= v[k]->tstamp) break; - std::swap(v[k],v[j]); - v[k]->idx = k; - v[j]->idx = j; + ret.reset(new TPCollWrapper(key,revision)); } - } - - TPCollCache - ::TPCollCache(size_t capacity) - { - m_history.reserve(capacity); - } - - TPCollWrapper* - TPCollCache - ::encache(TPCollWrapper* const& ptr) - { - using namespace boost; - // update time stamp: -#if defined(timespec) - clock_gettime(CLOCK_MONOTONIC, &ptr->tstamp); -#else - gettimeofday(&ptr->tstamp, NULL); -#endif - unique_lock<shared_mutex> lock(m_history_lock); - if (m_history.capacity() > 1) + if (m_cache.size() == 1) { - vector<TPCollWrapper*>& v = m_history; - if (ptr->idx >= 0) // ptr is already in history - { - assert(ptr == v[ptr->idx]); - size_t k = 2 * (ptr->idx + 1); - if (k < v.size()) bubble_up(v,k--); - if (k < v.size()) bubble_up(v,k); - } - else if (v.size() < v.capacity()) - { - size_t k = ptr->idx = v.size(); - v.push_back(ptr); - bubble_up(v,k); - } - else // someone else needs to go - { - v[0]->idx = -1; - release(v[0]); - v[0] = ptr; - bubble_down(v,0); - } + m_qfirst = m_qlast = foo.first; + ret->prev = m_cache.end(); } - return ptr; - } // TPCollCache::encache(...) - - TPCollWrapper* - TPCollCache - ::get(uint64_t key, size_t revision) - { - using namespace boost; - cache_t::iterator m; - { - shared_lock<shared_mutex> lock(m_cache_lock); - m = m_cache.find(key); - if (m == m_cache.end() || m->second->revision != revision) - return NULL; - ++m->second->refCount; - } - - encache(m->second); - // return NULL; - return m->second; - } // TPCollCache::get(...) - - void - TPCollCache - ::add(uint64_t key, TPCollWrapper* ptr) - { - { - boost::unique_lock<boost::shared_mutex> lock(m_cache_lock); - m_cache[key] = ptr; - ++ptr->refCount; - // ++m_tpc_ctr; - } - encache(ptr); - } // TPCollCache::add(...) - - void - TPCollCache - ::release(TPCollWrapper*& ptr) - { - if (!ptr) return; - - boost::upgrade_lock<boost::shared_mutex> lock(m_cache_lock); - - if (--ptr->refCount || ptr->idx >= 0) // tpc is still in use + else if (m_qlast != foo.first) { - ptr = NULL; - return; + ret->prev = m_qlast; + m_qlast->second->next = foo.first; + m_qlast = foo.first; } + ret->next = m_cache.end(); #if 0 - timespec t; clock_gettime(CLOCK_MONOTONIC,&t); - timespec r; clock_getres(CLOCK_MONOTONIC,&r); - float delta = t.tv_sec - ptr->tstamp.tv_sec; - cerr << "deleting old cache entry after " << delta << " seconds." - << " clock resolution is " << r.tv_sec << ":" << r.tv_nsec - << " at " << __FILE__ << ":" << __LINE__ << endl; + std::cerr << "AFTER" << std::endl; + ctr=0; + for (cache_t::iterator m = m_qfirst; m != m_cache.end(); m = m->second->next) + { + std::cerr << ++ctr << "/" << m_cache.size() << " " + << (m->second->key == key ? "*" : " ") + << m->second->key << " " + << m->second.use_count(); + if (m->second->prev != m_cache.end()) + std::cerr << " => " << m->second->prev->second->key; + std::cerr << std::endl; + } + std::cerr << "\n" << std::endl; #endif - cache_t::iterator m = m_cache.find(ptr->key); - if (m != m_cache.end() && m->second == ptr) - { // the cache could have been updated with a new pointer - // for the same phrase already, so we need to check - // if the pointer we cound is the one we want to get rid of, - // hence the second check - boost::upgrade_to_unique_lock<boost::shared_mutex> xlock(lock); - m_cache.erase(m); + if (m_cache.size() > m_capacity) + { + // size_t ctr = 0; + // size_t oldsize = m_cache.size(); + while (m_cache.size() > m_capacity && m_qfirst->second.use_count() == 1) + { + m_qfirst = m_qfirst->second->next; + // std::cerr << "erasing " << ++ctr << "/" << m_cache.size() << " " + // << m_qfirst->second->key << std::endl; + m_cache.erase(m_qfirst->second->prev); + } + // if (oldsize > m_cache.size()) std::cerr << "\n" << std::endl; } - - - delete ptr; - - - - ptr = NULL; - } // TPCollCache::release(...) - + return ret; + } // TPCollCache::get(...) + TPCollWrapper:: - TPCollWrapper(size_t r, uint64_t k) - : revision(r), key(k), refCount(0), idx(-1) + TPCollWrapper(uint64_t key_, size_t revision_) + : revision(revision_), key(key_) { } TPCollWrapper:: ~TPCollWrapper() - { - assert(this->refCount == 0); - } + { } } // namespace diff --git a/moses/TranslationModel/UG/TargetPhraseCollectionCache.h b/moses/TranslationModel/UG/TargetPhraseCollectionCache.h index 269200647..50d23ebdc 100644 --- a/moses/TranslationModel/UG/TargetPhraseCollectionCache.h +++ b/moses/TranslationModel/UG/TargetPhraseCollectionCache.h @@ -2,61 +2,46 @@ #pragma once #include <time.h> #include "moses/TargetPhraseCollection.h" - +#include <boost/atomic.hpp> +#include "mm/ug_typedefs.h" namespace Moses { - class TPCollWrapper - // wrapper around TargetPhraseCollection that includes reference counts - // and a time stamp for least-recently-used caching of TargetPhraseCollection-s - : public TargetPhraseCollection - { - public: - size_t const revision; - // revison; gets changed when the underlying corpus in Mmsapt is updated - uint64_t const key; // phrase key - uint32_t refCount; // reference count -#if defined(timespec) // timespec is better, but not available everywhere - timespec tstamp; // last use -#else - timeval tstamp; // last use -#endif - int idx; // position in the history heap - TPCollWrapper(size_t r, uint64_t const k); - ~TPCollWrapper(); - }; + class TPCollWrapper; class TPCollCache { - typedef boost::unordered_map<uint64_t, TPCollWrapper*> cache_t; - typedef std::vector<TPCollWrapper*> history_t; - cache_t m_cache; // maps from phrase ids to target phrase collections - mutable history_t m_history; // heap of live items, least recently used one on top - - mutable boost::shared_mutex m_cache_lock; // locks m_cache - mutable boost::shared_mutex m_history_lock; // locks m_history - -#if 0 - // mutable size_t m_tpc_ctr; - // counter of all live item, for debugging. probably obsolete; was used - // to track memory leaks -#endif - - TPCollWrapper* encache(TPCollWrapper* const& ptr); - // updates time stamp and position in least-recently-used heap m_history - public: - TPCollCache(size_t capacity=1000); + // typedef boost::unordered_map<uint64_t, SPTR<TPCollWrapper> > cache_t; + typedef std::map<uint64_t, SPTR<TPCollWrapper> > cache_t; + private: + uint32_t m_capacity; // capacity of cache + cache_t m_cache; // maps from ids to items + cache_t::iterator m_qfirst, m_qlast; + mutable boost::shared_mutex m_lock; + public: + TPCollCache(size_t capacity=10000); - TPCollWrapper* + SPTR<TPCollWrapper> get(uint64_t key, size_t revision); - void - add(uint64_t key, TPCollWrapper* ptr); - - void - release(TPCollWrapper*& tpc); }; + // wrapper around TargetPhraseCollection with reference counting + // and additional members for caching purposes + class TPCollWrapper + : public TargetPhraseCollection + { + friend class TPCollCache; + friend class Mmsapt; + public: + TPCollCache::cache_t::iterator prev, next; + public: + mutable boost::shared_mutex lock; + size_t const revision; // rev. No. of the underlying corpus + uint64_t const key; // phrase key + TPCollWrapper(uint64_t const key, size_t const rev); + ~TPCollWrapper(); + }; } diff --git a/moses/TranslationModel/UG/mm/ug_bitext_sampler.h b/moses/TranslationModel/UG/mm/ug_bitext_sampler.h index 701f4aaff..79014e35a 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_sampler.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_sampler.h @@ -20,7 +20,7 @@ namespace sapt { -enum sampling_method { full_coverage, random_sampling, ranked_sampling }; +enum sampling_method { full_coverage, random_sampling, ranked_sampling, ranked_sampling2 }; typedef ttrack::Position TokenPosition; class CandidateSorter @@ -45,15 +45,16 @@ BitextSampler : public Moses::reference_counter // const members // SPTR<bitext const> const m_bitext; // keep bitext alive while I am // should be an - iptr<bitext const> const m_bitext; // keep bitext alive as long as I am - size_t const m_plen; // length of lookup phrase - bool const m_fwd; // forward or backward direction? - SPTR<tsa const> const m_root; // root of suffix array - char const* m_next; // current position - char const* m_stop; // end of search range - sampling_method const m_method; // look at all/random/ranked samples - SPTR<bias const> const m_bias; // bias over candidates - size_t const m_samples; // how many samples at most + iptr<bitext const> const m_bitext; // keep bitext alive as long as I am + size_t const m_plen; // length of lookup phrase + bool const m_fwd; // forward or backward direction? + SPTR<tsa const> const m_root; // root of suffix array + char const* m_next; // current position + char const* m_stop; // end of search range + sampling_method const m_method; // look at all/random/ranked samples + SPTR<bias const> const m_bias; // bias over candidates + size_t const m_samples; // how many samples at most + size_t const m_min_samples; // non-const members SPTR<pstats> m_stats; // destination for phrase stats size_t m_ctr; // number of samples considered @@ -74,7 +75,7 @@ public: BitextSampler(BitextSampler const& other); BitextSampler const& operator=(BitextSampler const& other); BitextSampler(bitext const* const bitext, typename bitext::iter const& phrase, - SPTR<SamplingBias const> const& bias, size_t const max_samples, + SPTR<SamplingBias const> const& bias, size_t const min_samples, size_t const max_samples, sampling_method const method); ~BitextSampler(); SPTR<pstats> stats(); @@ -173,7 +174,7 @@ template<typename Token> BitextSampler<Token>:: BitextSampler(Bitext<Token> const* const bitext, typename bitext::iter const& phrase, - SPTR<SamplingBias const> const& bias, size_t const max_samples, + SPTR<SamplingBias const> const& bias, size_t const min_samples, size_t const max_samples, sampling_method const method) : m_bitext(bitext) , m_plen(phrase.size()) @@ -184,6 +185,7 @@ BitextSampler(Bitext<Token> const* const bitext, , m_method(method) , m_bias(bias) , m_samples(max_samples) + , m_min_samples(min_samples) , m_ctr(0) , m_total_bias(0) , m_finished(false) @@ -208,6 +210,7 @@ BitextSampler(BitextSampler const& other) , m_method(other.m_method) , m_bias(other.m_bias) , m_samples(other.m_samples) + , m_min_samples(other.m_min_samples) , m_rnd(0) // , m_rnd_denom(m_rnd.max() + 1) { diff --git a/moses/TranslationModel/UG/mm/ug_sampling_bias.cc b/moses/TranslationModel/UG/mm/ug_sampling_bias.cc index 85e143278..f43e05288 100644 --- a/moses/TranslationModel/UG/mm/ug_sampling_bias.cc +++ b/moses/TranslationModel/UG/mm/ug_sampling_bias.cc @@ -89,6 +89,12 @@ namespace sapt return m_bias_map; } + const std::map<id_type, float>& + DocumentBias:: + GetDocumentBiasMap() const { + return m_bias; + } + void DocumentBias:: init_from_json diff --git a/moses/TranslationModel/UG/mm/ug_sampling_bias.h b/moses/TranslationModel/UG/mm/ug_sampling_bias.h index 3f8fd1fed..4dc68cd7f 100644 --- a/moses/TranslationModel/UG/mm/ug_sampling_bias.h +++ b/moses/TranslationModel/UG/mm/ug_sampling_bias.h @@ -73,6 +73,8 @@ namespace sapt size_t size() const; + + const std::map<id_type, float>& GetDocumentBiasMap() const; }; class diff --git a/moses/TranslationModel/UG/mm/ug_typedefs.h b/moses/TranslationModel/UG/mm/ug_typedefs.h index bba414273..fc9d7faef 100644 --- a/moses/TranslationModel/UG/mm/ug_typedefs.h +++ b/moses/TranslationModel/UG/mm/ug_typedefs.h @@ -33,7 +33,9 @@ namespace sapt typedef tpt::filepos_type filepos_type; } +#ifndef SPTR #define SPTR boost::shared_ptr +#endif #define iptr boost::intrusive_ptr #define scoptr boost::scoped_ptr #define rcast reinterpret_cast diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index a8bc0d525..180b66ba3 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -181,6 +181,9 @@ namespace Moses dflt = pair<string,string> ("sample","1000"); m_default_sample_size = atoi(param.insert(dflt).first->second.c_str()); + dflt = pair<string,string> ("min-sample","0"); + m_min_sample_size = atoi(param.insert(dflt).first->second.c_str()); + dflt = pair<string,string>("workers","0"); m_workers = atoi(param.insert(dflt).first->second.c_str()); if (m_workers == 0) m_workers = boost::thread::hardware_concurrency(); @@ -192,8 +195,9 @@ namespace Moses dflt = pair<string,string>("table-limit","20"); m_tableLimit = atoi(param.insert(dflt).first->second.c_str()); - dflt = pair<string,string>("cache","10000"); - m_cache_size = max(1000,atoi(param.insert(dflt).first->second.c_str())); + dflt = pair<string,string>("cache","100000"); + m_cache_size = max(10000,atoi(param.insert(dflt).first->second.c_str())); + m_cache.reset(new TPCollCache(m_cache_size)); // m_history.reserve(hsize); // in plain language: cache size is at least 1000, and 10,000 by default @@ -249,6 +253,8 @@ namespace Moses m_sampling_method = random_sampling; else if (m->second == "ranked") m_sampling_method = ranked_sampling; + else if (m->second == "ranked2") + m_sampling_method = ranked_sampling2; else if (m->second == "full") m_sampling_method = full_coverage; else UTIL_THROW2("unrecognized specification 'method='" << m->second @@ -297,6 +303,7 @@ namespace Moses known_parameters.push_back("prov"); known_parameters.push_back("rare"); known_parameters.push_back("sample"); + known_parameters.push_back("min-sample"); known_parameters.push_back("smooth"); known_parameters.push_back("table-limit"); known_parameters.push_back("tuneable"); @@ -618,29 +625,32 @@ namespace Moses { InputPath &inputPath = **iter; const Phrase &phrase = inputPath.GetPhrase(); - const TargetPhraseCollection *targetPhrases + TargetPhraseCollection::shared_ptr targetPhrases = this->GetTargetPhraseCollectionLEGACY(ttask,phrase); inputPath.SetTargetPhrases(*this, targetPhrases, NULL); } } - TargetPhraseCollection const* - Mmsapt:: - GetTargetPhraseCollectionLEGACY(const Phrase& src) const - { - UTIL_THROW2("Don't call me without the translation task."); - } + // TargetPhraseCollection::shared_ptr + // Mmsapt:: + // GetTargetPhraseCollectionLEGACY(const Phrase& src) const + // { + // UTIL_THROW2("Don't call me without the translation task."); + // } // This is not the most efficient way of phrase lookup! - TargetPhraseCollection const* + TargetPhraseCollection::shared_ptr Mmsapt:: GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) const { + SPTR<TPCollWrapper> ret; + // boost::unique_lock<boost::shared_mutex> xlock(m_lock); + // map from Moses Phrase to internal id sequence vector<id_type> sphrase; fillIdSeq(src, m_ifactor, *(btfix->V1), sphrase); - if (sphrase.size() == 0) return NULL; - + if (sphrase.size() == 0) return ret; + // Reserve a local copy of the dynamic bitext in its current form. /btdyn/ // is set to a new copy of the dynamic bitext every time a sentence pair // is added. /dyn/ keeps the old bitext around as long as we need it. @@ -655,31 +665,42 @@ namespace Moses // lookup phrases in both bitexts TSA<Token>::tree_iterator mfix(btfix->I1.get(), &sphrase[0], sphrase.size()); TSA<Token>::tree_iterator mdyn(dyn->I1.get()); - if (dyn->I1.get()) + if (dyn->I1.get()) // we have a dynamic bitext for (size_t i = 0; mdyn.size() == i && i < sphrase.size(); ++i) mdyn.extend(sphrase[i]); if (mdyn.size() != sphrase.size() && mfix.size() != sphrase.size()) - return NULL; // phrase not found in either bitext + return ret; // phrase not found in either bitext // do we have cached results for this phrase? uint64_t phrasekey = (mfix.size() == sphrase.size() - ? (mfix.getPid()<<1) : (mdyn.getPid()<<1)+1); + ? (mfix.getPid()<<1) + : (mdyn.getPid()<<1)+1); // get context-specific cache of items previously looked up SPTR<ContextScope> const& scope = ttask->GetScope(); SPTR<TPCollCache> cache = scope->get<TPCollCache>(cache_key); - if (!cache) cache = m_cache; - TPCollWrapper* ret = cache->get(phrasekey, dyn->revision()); - // TO DO: we should revise the revision mechanism: we take the length - // of the dynamic bitext (in sentences) at the time the PT entry - // was stored as the time stamp. For each word in the + if (!cache) cache = m_cache; // no context-specific cache, use global one + + ret = cache->get(phrasekey, dyn->revision()); + // TO DO: we should revise the revision mechanism: we take the + // length of the dynamic bitext (in sentences) at the time the PT + // entry was stored as the time stamp. For each word in the // vocabulary, we also store its most recent occurrence in the // bitext. Only if the timestamp of each word in the phrase is // newer than the timestamp of the phrase itself we must update // the entry. - if (ret) return ret; // yes, was cached => DONE + // std::cerr << "Phrasekey is " << ret->key << " at " << HERE << std::endl; + // std::cerr << ret << " with " << ret->refCount << " references at " + // << HERE << std::endl; + boost::upgrade_lock<boost::shared_mutex> rlock(ret->lock); + if (ret->GetSize()) return ret; + + // new TPC (not found or old one was not up to date) + boost::upgrade_to_unique_lock<boost::shared_mutex> wlock(rlock); + // maybe another thread did the work while we waited for the lock ? + if (ret->GetSize()) return ret; // OK: pt entry NOT found or NOT up to date // lookup and expansion could be done in parallel threads, @@ -697,12 +718,16 @@ namespace Moses else { BitextSampler<Token> s(btfix.get(), mfix, context->bias, - m_default_sample_size, m_sampling_method); + m_min_sample_size, + m_default_sample_size, + m_sampling_method); s(); sfix = s.stats(); } } - if (mdyn.size() == sphrase.size()) sdyn = dyn->lookup(ttask, mdyn); + + if (mdyn.size() == sphrase.size()) + sdyn = dyn->lookup(ttask, mdyn); vector<PhrasePair<Token> > ppfix,ppdyn; PhrasePair<Token>::SortByTargetIdSeq sort_by_tgt_id; @@ -716,8 +741,8 @@ namespace Moses expand(mdyn, *dyn, *sdyn, ppdyn, m_bias_log); sort(ppdyn.begin(), ppdyn.end(),sort_by_tgt_id); } + // now we have two lists of Phrase Pairs, let's merge them - ret = new TPCollWrapper(dyn->revision(), phrasekey); PhrasePair<Token>::SortByTargetIdSeq sorter; size_t i = 0; size_t k = 0; while (i < ppfix.size() && k < ppdyn.size()) @@ -729,6 +754,8 @@ namespace Moses } while (i < ppfix.size()) ret->Add(mkTPhrase(ttask,src,&ppfix[i++],NULL,dyn)); while (k < ppdyn.size()) ret->Add(mkTPhrase(ttask,src,NULL,&ppdyn[k++],dyn)); + + // Pruning should not be done here but outside! if (m_tableLimit) ret->Prune(true, m_tableLimit); else ret->Prune(true,ret->GetSize()); @@ -744,7 +771,6 @@ namespace Moses } } #endif - cache->add(phrasekey, ret); return ret; } @@ -779,14 +805,27 @@ namespace Moses void Mmsapt:: - set_bias_via_server(ttasksptr const& ttask) + setup_bias(ttasksptr const& ttask) { + + std::cerr << "Setting up bias at " << HERE << std::endl; + SPTR<ContextScope> const& scope = ttask->GetScope(); SPTR<ContextForQuery> context = scope->get<ContextForQuery>(btfix.get(), true); - if (m_bias_server.size() && context->bias == NULL && ttask->GetContextWindow()) - { // we need to create the bias - boost::unique_lock<boost::shared_mutex> lock(context->lock); - // string const& context_words = ttask->GetContextString(); + if (context->bias) return; + // boost::unique_lock<boost::shared_mutex> ctxlock(context->lock); + + // bias weights specified with the session? + SPTR<std::map<std::string, float> const> w = ttask->GetContextWeights(); + if (w && !w->empty()) + { + if (m_bias_log) + *m_bias_log << "BIAS WEIGHTS GIVEN WITH INPUT at " << HERE << endl; + context->bias = btfix->SetupDocumentBias(*w, m_bias_log); + } + else if (m_bias_server.size() && ttask->GetContextWindow()) + { + std::cerr << "via server at " << HERE << std::endl; string context_words; BOOST_FOREACH(string const& line, *ttask->GetContextWindow()) { @@ -796,67 +835,50 @@ namespace Moses if (context_words.size()) { if (m_bias_log) - { - *m_bias_log << HERE << endl << "BIAS LOOKUP CONTEXT: " - << context_words << endl; - context->bias_log = m_bias_log; - } + *m_bias_log << "GETTING BIAS FROM SERVER at " << HERE << endl + << "BIAS LOOKUP CONTEXT: " << context_words << endl; context->bias = btfix->SetupDocumentBias(m_bias_server, context_words, m_bias_log); - context->bias->loglevel = m_bias_loglevel; - context->bias->log = m_bias_log; //Reset the bias in the ttaskptr so that other functions //so that other functions can utilize the biases; ttask->ReSetContextWeights(context->bias->getBiasMap()); } - // if (!context->cache1) context->cache1.reset(new pstats::cache_t); - // if (!context->cache2) context->cache2.reset(new pstats::cache_t); } - else if (!ttask->GetContextWeights().empty()) + if (context->bias) { - if (m_bias_log) - { - *m_bias_log << HERE << endl - << "BIAS FROM MAP LOOKUP" << endl; - context->bias_log = m_bias_log; - } - context->bias - = btfix->SetupDocumentBias(ttask->GetContextWeights(), m_bias_log); + context->bias_log = m_bias_log; context->bias->loglevel = m_bias_loglevel; - context->bias->log = m_bias_log; - // if (!context->cache1) context->cache1.reset(new pstats::cache_t); - // if (!context->cache2) context->cache2.reset(new pstats::cache_t); } - if (!context->cache1) context->cache1.reset(new pstats::cache_t); - if (!context->cache2) context->cache2.reset(new pstats::cache_t); } void Mmsapt:: InitializeForInput(ttasksptr const& ttask) { + boost::unique_lock<boost::shared_mutex> mylock(m_lock); + SPTR<ContextScope> const& scope = ttask->GetScope(); + SPTR<TPCollCache> localcache = scope->get<TPCollCache>(cache_key); SPTR<ContextForQuery> context = scope->get<ContextForQuery>(btfix.get(), true); + boost::unique_lock<boost::shared_mutex> ctxlock(context->lock); - // set sampling bias, depending on sampling method specified -#if 0 - // for the time being, always use the external bias - if (m_sampling_method == random_sampling) - set_bias_via_server(ttask); - else UTIL_THROW2("Unknown sampling method: " << m_sampling_method); -#else - set_bias_via_server(ttask); -#endif - - boost::unique_lock<boost::shared_mutex> mylock(m_lock); - SPTR<TPCollCache> localcache = scope->get<TPCollCache>(cache_key); + if (localcache) std::cerr << "have local cache " << std::endl; + std::cerr << "BOO at " << HERE << std::endl; if (!localcache) { - if (context->bias) localcache.reset(new TPCollCache(m_cache_size)); + std::cerr << "no local cache at " << HERE << std::endl; + setup_bias(ttask); + if (context->bias) + { + localcache.reset(new TPCollCache(m_cache_size)); + } else localcache = m_cache; scope->set<TPCollCache>(cache_key, localcache); } + if (!context->cache1) context->cache1.reset(new pstats::cache_t); + if (!context->cache2) context->cache2.reset(new pstats::cache_t); + if (m_lr_func_name.size() && m_lr_func == NULL) { FeatureFunction* lr = &FeatureFunction::FindFeatureFunction(m_lr_func_name); @@ -867,13 +889,6 @@ namespace Moses } } - // bool - // Mmsapt:: - // PrefixExists(Moses::Phrase const& phrase) const - // { - // return PrefixExists(phrase,NULL); - // } - bool Mmsapt:: PrefixExists(ttasksptr const& ttask, Moses::Phrase const& phrase) const @@ -892,7 +907,7 @@ namespace Moses if (!context->cache1->get(pid)) { BitextSampler<Token> s(btfix.get(), mfix, context->bias, - m_default_sample_size, m_sampling_method); + m_min_sample_size, m_default_sample_size, m_sampling_method); if (*context->cache1->get(pid, s.stats()) == s.stats()) m_thread_pool->add(s); } @@ -918,15 +933,26 @@ namespace Moses return mdyn.size() == myphrase.size(); } +#if 0 void Mmsapt - ::Release(ttasksptr const& ttask, TargetPhraseCollection*& tpc) const + ::Release(ttasksptr const& ttask, TargetPhraseCollection::shared_ptr*& tpc) const { + if (!tpc) + { + // std::cerr << "NULL pointer at " << HERE << std::endl; + return; + } SPTR<TPCollCache> cache = ttask->GetScope()->get<TPCollCache>(cache_key); - TPCollWrapper* foo = static_cast<TPCollWrapper*>(tpc); - if (cache) cache->release(foo); + + TPCollWrapper const* foo = static_cast<TPCollWrapper const*>(tpc); + + // std::cerr << "\nReleasing " << foo->key << "\n" << std::endl; + + if (cache) cache->release(static_cast<TPCollWrapper const*>(tpc)); tpc = NULL; } +#endif bool Mmsapt ::ProvidesPrefixCheck() const { return true; } diff --git a/moses/TranslationModel/UG/mmsapt.h b/moses/TranslationModel/UG/mmsapt.h index d8ebd06ea..0dc8f575f 100644 --- a/moses/TranslationModel/UG/mmsapt.h +++ b/moses/TranslationModel/UG/mmsapt.h @@ -75,6 +75,7 @@ namespace Moses // alpha parameter for lexical smoothing (joint+alpha)/(marg + alpha) // must be > 0 if dynamic size_t m_default_sample_size; + size_t m_min_sample_size; size_t m_workers; // number of worker threads for sampling the bitexts std::vector<std::string> m_feature_set_names; // one or more of: standard, datasource std::string m_bias_logfile; @@ -147,7 +148,7 @@ namespace Moses std::vector<FactorType> m_ifactor, m_ofactor; void setup_local_feature_functions(); - void set_bias_via_server(ttasksptr const& ttask); + void setup_bias(ttasksptr const& ttask); #if PROVIDES_RANKED_SAMPLING void @@ -178,7 +179,7 @@ namespace Moses uint64_t const pid1, sapt::pstats const& stats, sapt::Bitext<Token> const & bt, - TargetPhraseCollection* tpcoll + TargetPhraseCollection::shared_ptr tpcoll ) const; bool @@ -186,14 +187,14 @@ namespace Moses (Phrase const& src, uint64_t const pid1a, sapt::pstats * statsa, sapt::Bitext<Token> const & bta, uint64_t const pid1b, sapt::pstats const* statsb, sapt::Bitext<Token> const & btb, - TargetPhraseCollection* tpcoll) const; + TargetPhraseCollection::shared_ptr tpcoll) const; bool combine_pstats (Phrase const& src, uint64_t const pid1a, sapt::pstats* statsa, sapt::Bitext<Token> const & bta, uint64_t const pid1b, sapt::pstats const* statsb, sapt::Bitext<Token> const & btb, - TargetPhraseCollection* tpcoll) const; + TargetPhraseCollection::shared_ptr tpcoll) const; void load_extra_data(std::string bname, bool locking); void load_bias(std::string bname); @@ -208,15 +209,15 @@ namespace Moses std::string const& GetName() const; #ifndef NO_MOSES - TargetPhraseCollection const* + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) const; - TargetPhraseCollection const* - GetTargetPhraseCollectionLEGACY(const Phrase& src) const; + // TargetPhraseCollection::shared_ptr + // GetTargetPhraseCollectionLEGACY(const Phrase& src) const; void - GetTargetPhraseCollectionBatch(ttasksptr const& ttask, - const InputPathList &inputPathQueue) const; + GetTargetPhraseCollectionBatch + (ttasksptr const& ttask, InputPathList const& inputPathQueue) const; //! Create a sentence-specific manager for SCFG rule lookup. ChartRuleLookupManager* @@ -233,7 +234,8 @@ namespace Moses void setWeights(std::vector<float> const& w); - void Release(ttasksptr const& ttask, TargetPhraseCollection*& tpc) const; + // void Release(ttasksptr const& ttask, + // TargetPhraseCollection const*& tpc) const; // some consumer lets me know that *tpc isn't needed any more diff --git a/moses/TranslationModel/UG/ptable-lookup.cc b/moses/TranslationModel/UG/ptable-lookup.cc index cccc5b857..d37097c97 100644 --- a/moses/TranslationModel/UG/ptable-lookup.cc +++ b/moses/TranslationModel/UG/ptable-lookup.cc @@ -80,7 +80,8 @@ int main(int argc, char* argv[]) Phrase& p = *phrase; cout << p << endl; - TargetPhraseCollection const* trg = PT->GetTargetPhraseCollectionLEGACY(ttask,p); + TargetPhraseCollection::shared_ptr trg + = PT->GetTargetPhraseCollectionLEGACY(ttask,p); if (!trg) continue; vector<size_t> order(trg->GetSize()); for (size_t i = 0; i < order.size(); ++i) order[i] = i; @@ -118,7 +119,7 @@ int main(int argc, char* argv[]) } cout << endl; } - PT->Release(trg); + // PT->Release(ttask, trg); } exit(0); } diff --git a/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h b/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h index 25b81e56a..b195be290 100644 --- a/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h +++ b/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h @@ -2,8 +2,10 @@ // Phrase scorer that records the aggregated bias score // +#include "util/exception.hh" #include "sapt_pscore_base.h" #include <boost/dynamic_bitset.hpp> +#include <cstdio> namespace sapt { @@ -11,12 +13,14 @@ namespace sapt { class PScoreCumBias : public PhraseScorer<Token> { + float m_floor; public: PScoreCumBias(std::string const spec) { this->m_index = -1; this->m_feature_names.push_back("cumb"); this->m_num_feats = this->m_feature_names.size(); + this->m_floor = std::atof(spec.c_str()); } bool @@ -28,7 +32,7 @@ namespace sapt { std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; - (*dest)[this->m_index] = log(pp.cum_bias); + (*dest)[this->m_index] = log(std::max(m_floor,pp.cum_bias)); } }; } // namespace sapt diff --git a/moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp b/moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp index 9003817d8..eabdd8753 100644 --- a/moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp +++ b/moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp @@ -7,13 +7,14 @@ // #include <iostream> +#include "util/string_stream.hh" #include "SentenceAlignment.h" namespace tmmt { std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const { - std::stringstream strme; + util::StringStream strme; for (size_t i = 0; i < target.size(); ++i) { const WORD &word = vocab.GetWord(target[i]); strme << word << " "; diff --git a/moses/TranslationModel/fuzzy-match/SentenceAlignment.h b/moses/TranslationModel/fuzzy-match/SentenceAlignment.h index a777c1eb0..4d6dc430c 100644 --- a/moses/TranslationModel/fuzzy-match/SentenceAlignment.h +++ b/moses/TranslationModel/fuzzy-match/SentenceAlignment.h @@ -12,6 +12,7 @@ #include <sstream> #include <vector> #include "Vocabulary.h" +#include "util/string_stream.hh" namespace tmmt { @@ -27,7 +28,7 @@ struct SentenceAlignment { std::string getTargetString(const Vocabulary &vocab) const; std::string getAlignmentString() const { - std::stringstream strme; + util::StringStream strme; for (size_t i = 0; i < alignment.size(); ++i) { const std::pair<int,int> &alignPair = alignment[i]; strme << alignPair.first << "-" << alignPair.second << " "; diff --git a/moses/TranslationOptionCollection.cpp b/moses/TranslationOptionCollection.cpp index 07544b88d..f0f6c8d84 100644 --- a/moses/TranslationOptionCollection.cpp +++ b/moses/TranslationOptionCollection.cpp @@ -410,7 +410,7 @@ CreateTranslationOptionsForRange const DecodeStep &dstep = **d; const PhraseDictionary &pdict = *dstep.GetPhraseDictionaryFeature(); - const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(pdict); + TargetPhraseCollection::shared_ptr targetPhrases = inputPath.GetTargetPhrases(pdict); static_cast<const Tstep&>(dstep).ProcessInitialTranslation (m_source, *oldPtoc, sPos, ePos, adhereTableLimit, inputPath, targetPhrases); @@ -431,7 +431,7 @@ CreateTranslationOptionsForRange TranslationOption &inputPartialTranslOpt = **pto; if (const Tstep *tstep = dynamic_cast<const Tstep*>(dstep)) { const PhraseDictionary &pdict = *tstep->GetPhraseDictionaryFeature(); - const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(pdict); + TargetPhraseCollection::shared_ptr targetPhrases = inputPath.GetTargetPhrases(pdict); tstep->Process(inputPartialTranslOpt, *dstep, *newPtoc, this, adhereTableLimit, targetPhrases); } else { @@ -483,14 +483,14 @@ SetInputScore(const InputPath &inputPath, PartialTranslOptColl &oldPtoc) const ScorePair* inputScore = inputPath.GetInputScore(); if (inputScore == NULL) return; - const InputFeature &inputFeature = InputFeature::Instance(); + const InputFeature *inputFeature = InputFeature::InstancePtr(); const std::vector<TranslationOption*> &transOpts = oldPtoc.GetList(); for (size_t i = 0; i < transOpts.size(); ++i) { TranslationOption &transOpt = *transOpts[i]; ScoreComponentCollection &scores = transOpt.GetScoreBreakdown(); - scores.PlusEquals(&inputFeature, *inputScore); + scores.PlusEquals(inputFeature, *inputScore); } } diff --git a/moses/TranslationOptionCollectionConfusionNet.cpp b/moses/TranslationOptionCollectionConfusionNet.cpp index 6ee83d969..391d1d6b6 100644 --- a/moses/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/TranslationOptionCollectionConfusionNet.cpp @@ -35,8 +35,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, BOOST_FOREACH(PhraseDictionary* pd, PhraseDictionary::GetColl()) if (pd->ProvidesPrefixCheck()) prefixCheckers.push_back(pd); - const InputFeature &inputFeature = InputFeature::Instance(); - UTIL_THROW_IF2(&inputFeature == NULL, "Input feature must be specified"); + const InputFeature *inputFeature = InputFeature::InstancePtr(); + UTIL_THROW_IF2(inputFeature == NULL, "Input feature must be specified"); size_t inputSize = input.GetSize(); m_inputPathMatrix.resize(inputSize); @@ -62,7 +62,7 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ScorePair &scores = col[i].second; ScorePair *inputScore = new ScorePair(scores); - InputPath *path = new InputPath(subphrase, labels, range, NULL, inputScore); + InputPath *path = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); list.push_back(path); m_inputPathQueue.push_back(path); @@ -113,7 +113,7 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, ScorePair *inputScore = new ScorePair(*prevInputScore); inputScore->PlusEquals(scores); - InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore); + InputPath *path = new InputPath(ttask, subphrase, labels, range, &prevPath, inputScore); list.push_back(path); m_inputPathQueue.push_back(path); @@ -143,7 +143,7 @@ InputPathList &TranslationOptionCollectionConfusionNet::GetInputPathList(size_t */ void TranslationOptionCollectionConfusionNet::ProcessUnknownWord(size_t sourcePos) { - ConfusionNet const& source=dynamic_cast<ConfusionNet const&>(m_source); + ConfusionNet const& source=static_cast<ConfusionNet const&>(m_source); ConfusionNet::Column const& coll=source.GetColumn(sourcePos); const InputPathList &inputPathList = GetInputPathList(sourcePos, sourcePos); diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index fde40e538..2cb3b6387 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -28,8 +28,8 @@ TranslationOptionCollectionLattice UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(), "Not for models using the legqacy binary phrase table"); - const InputFeature &inputFeature = InputFeature::Instance(); - UTIL_THROW_IF2(&inputFeature == NULL, "Input feature must be specified"); + const InputFeature *inputFeature = InputFeature::InstancePtr(); + UTIL_THROW_IF2(inputFeature == NULL, "Input feature must be specified"); size_t maxPhraseLength = StaticData::Instance().GetMaxPhraseLength(); size_t size = input.GetSize(); @@ -61,7 +61,8 @@ TranslationOptionCollectionLattice const ScorePair &scores = col[i].second; ScorePair *inputScore = new ScorePair(scores); - InputPath *path = new InputPath(subphrase, labels, range, NULL, inputScore); + InputPath *path + = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); path->SetNextNode(nextNode); m_inputPathQueue.push_back(path); @@ -113,7 +114,8 @@ void TranslationOptionCollectionLattice::Extend(const InputPath &prevPath, const ScorePair *inputScore = new ScorePair(*prevInputScore); inputScore->PlusEquals(scores); - InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore); + InputPath *path = new InputPath(prevPath.ttask, subphrase, labels, + range, &prevPath, inputScore); path->SetNextNode(nextNode); m_inputPathQueue.push_back(path); @@ -140,7 +142,8 @@ void TranslationOptionCollectionLattice::CreateTranslationOptions() for (size_t i = 0; i < m_inputPathQueue.size(); ++i) { const InputPath &path = *m_inputPathQueue[i]; - const TargetPhraseCollection *tpColl = path.GetTargetPhrases(phraseDictionary); + TargetPhraseCollection::shared_ptr tpColl + = path.GetTargetPhrases(phraseDictionary); const WordsRange &range = path.GetWordsRange(); if (tpColl && tpColl->GetSize()) { diff --git a/moses/TranslationOptionCollectionText.cpp b/moses/TranslationOptionCollectionText.cpp index 354d09f47..01b22a788 100644 --- a/moses/TranslationOptionCollectionText.cpp +++ b/moses/TranslationOptionCollectionText.cpp @@ -48,11 +48,11 @@ TranslationOptionCollectionText::TranslationOptionCollectionText(ttasksptr const InputPath *path; if (range.GetNumWordsCovered() == 1) { - path = new InputPath(subphrase, labels, range, NULL, NULL); + path = new InputPath(ttask, subphrase, labels, range, NULL, NULL); vec.push_back(path); } else { const InputPath &prevPath = GetInputPath(startPos, endPos - 1); - path = new InputPath(subphrase, labels, range, &prevPath, NULL); + path = new InputPath(ttask, subphrase, labels, range, &prevPath, NULL); vec.push_back(path); } @@ -75,7 +75,7 @@ void TranslationOptionCollectionText::ProcessUnknownWord(size_t sourcePos) */ bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t startPosition, size_t endPosition) const { - Sentence const& source=dynamic_cast<Sentence const&>(m_source); + Sentence const& source=static_cast<Sentence const&>(m_source); return source.XmlOverlap(startPosition,endPosition); } @@ -85,7 +85,7 @@ bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t start bool TranslationOptionCollectionText::ViolatesXmlOptionsConstraint(size_t startPosition, size_t endPosition, TranslationOption *transOpt) const { // skip if there is no overlap - Sentence const& source=dynamic_cast<Sentence const&>(m_source); + Sentence const& source=static_cast<Sentence const&>(m_source); if (!source.XmlOverlap(startPosition,endPosition)) { return false; } @@ -135,7 +135,7 @@ bool TranslationOptionCollectionText::ViolatesXmlOptionsConstraint(size_t startP */ void TranslationOptionCollectionText::CreateXmlOptionsForRange(size_t startPos, size_t endPos) { - Sentence const& source=dynamic_cast<Sentence const&>(m_source); + Sentence const& source=static_cast<Sentence const&>(m_source); InputPath &inputPath = GetInputPath(startPos,endPos); vector <TranslationOption*> xmlOptions; diff --git a/moses/TranslationTask.cpp b/moses/TranslationTask.cpp index df77b60c7..a29e223f5 100644 --- a/moses/TranslationTask.cpp +++ b/moses/TranslationTask.cpp @@ -30,7 +30,7 @@ GetContextWindow() const return m_context; } -std::map<std::string, float> const& +SPTR<std::map<std::string, float> const> TranslationTask::GetContextWeights() const { return m_context_weights; @@ -40,7 +40,7 @@ void TranslationTask ::ReSetContextWeights(std::map<std::string, float> const& new_weights) { - m_context_weights = new_weights; + m_context_weights.reset(new std::map<string,float>(new_weights)); } void @@ -54,11 +54,13 @@ void TranslationTask:: SetContextWeights(std::string const& context_weights) { + SPTR<std::map<string,float> > foo(new std::map<string,float>); std::vector<std::string> tokens = Tokenize(context_weights,":"); for (std::vector<std::string>::iterator it = tokens.begin(); it != tokens.end(); it++) { std::vector<std::string> key_and_value = Tokenize(*it, ","); - m_context_weights.insert(std::pair<std::string, float>(key_and_value[0], atof(key_and_value[1].c_str()))); + foo->insert(std::pair<std::string, float>(key_and_value[0], atof(key_and_value[1].c_str()))); } + m_context_weights = foo; } diff --git a/moses/TranslationTask.h b/moses/TranslationTask.h index 211ade610..288e9a3ce 100644 --- a/moses/TranslationTask.h +++ b/moses/TranslationTask.h @@ -67,7 +67,7 @@ protected: // task stays alive till it's done with it. boost::shared_ptr<std::vector<std::string> > m_context; - std::map<std::string, float> m_context_weights; + SPTR<std::map<std::string, float> > m_context_weights; public: boost::shared_ptr<TranslationTask> @@ -130,7 +130,7 @@ public: void SetContextWindow(boost::shared_ptr<std::vector<std::string> > const& cw); - std::map<std::string, float> const& GetContextWeights() const; + SPTR<std::map<std::string, float> const> GetContextWeights() const; void SetContextWeights(std::string const& context_weights); void ReSetContextWeights(std::map<std::string, float> const& new_weights); diff --git a/moses/TypeDef.h b/moses/TypeDef.h index 4ccc25143..18f01f73c 100644 --- a/moses/TypeDef.h +++ b/moses/TypeDef.h @@ -189,3 +189,4 @@ typedef boost::shared_ptr<TranslationTask> ttasksptr; typedef boost::weak_ptr<TranslationTask> ttaskwptr; } +#define SPTR boost::shared_ptr diff --git a/moses/Util.h b/moses/Util.h index 58152f7ce..926f5c32b 100644 --- a/moses/Util.h +++ b/moses/Util.h @@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <cstdlib> #include <cstring> #include "util/exception.hh" +#include "util/string_stream.hh" #include "TypeDef.h" namespace Moses @@ -343,7 +344,7 @@ inline std::vector<std::string> TokenizeFirstOnly(const std::string& str, template <typename T> std::string Join(const std::string& delimiter, const std::vector<T>& items) { - std::ostringstream outstr; + util::StringStream outstr; if(items.size() == 0) return ""; outstr << items[0]; for(unsigned int i = 1; i < items.size(); i++) @@ -357,7 +358,7 @@ std::string Join(const std::string& delimiter, const std::vector<T>& items) template<typename It> std::string Join(const std::string &delim, It begin, It end) { - std::ostringstream outstr; + util::StringStream outstr; if (begin != end) outstr << *begin++; for ( ; begin != end; ++begin) @@ -536,6 +537,27 @@ class FeatureFunction; void PrintFeatureWeight(const FeatureFunction* ff); void ShowWeights(); +template<typename T> +class UnorderedComparer +{ +public: + size_t operator()(const T& obj) const { + return obj.hash(); + } + + bool operator()(const T& a, const T& b) const { + return a == b; + } + + size_t operator()(const T* obj) const { + return obj->hash(); + } + + bool operator()(const T* a, const T* b) const { + return (*a) == (*b); + } + +}; } // namespace diff --git a/moses/Word.cpp b/moses/Word.cpp index f55be5ee8..a1c852f7c 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "FactorCollection.h" #include "StaticData.h" // needed to determine the FactorDelimiter #include "util/exception.hh" +#include "util/string_stream.hh" #include "util/tokenize_piece.hh" using namespace std; @@ -63,7 +64,23 @@ int Word::Compare(const Word &targetWord, const Word &sourceWord) return (targetFactor<sourceFactor) ? -1 : +1; } return 0; +} +bool Word::operator==(const Word &compare) const +{ + if (IsNonTerminal() != compare.IsNonTerminal()) { + return false; + } + + for (size_t factorType = 0 ; factorType < MAX_NUM_FACTORS ; factorType++) { + const Factor *thisFactor = GetFactor(factorType); + const Factor *otherFactor = compare.GetFactor(factorType); + + if (thisFactor != otherFactor) { + return false; + } + } + return true; } void Word::Merge(const Word &sourceWord) @@ -79,7 +96,7 @@ void Word::Merge(const Word &sourceWord) std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const { - stringstream strme; + util::StringStream strme; const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); bool firstPass = true; unsigned int stop = min(max_fax(),factorType.size()); @@ -195,7 +212,7 @@ TO_STRING_BODY(Word); // friend ostream& operator<<(ostream& out, const Word& word) { - stringstream strme; + util::StringStream strme; const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); bool firstPass = true; unsigned int stop = max_fax(); @@ -208,7 +225,7 @@ ostream& operator<<(ostream& out, const Word& word) } else { strme << factorDelimiter; } - strme << *factor; + strme << factor->GetString(); } } out << strme.str() << " "; diff --git a/moses/Word.h b/moses/Word.h index efdebddfa..bac69dd0f 100644 --- a/moses/Word.h +++ b/moses/Word.h @@ -116,30 +116,12 @@ public: StringPiece GetString(FactorType factorType) const; TO_STRING(); - //! transitive comparison of Word objects - inline bool operator< (const Word &compare) const { - // needed to store word in GenerationDictionary map - // uses comparison of FactorKey - // 'proper' comparison, not address/id comparison - return Compare(*this, compare) < 0; - } - - inline bool operator== (const Word &compare) const { - // needed to store word in GenerationDictionary map - // uses comparison of FactorKey - // 'proper' comparison, not address/id comparison - return Compare(*this, compare) == 0; - } + bool operator== (const Word &compare) const; inline bool operator!= (const Word &compare) const { - return Compare(*this, compare) != 0; - } - - int Compare(const Word &other) const { - return Compare(*this, other); + return !(*this == compare); } - /* static functions */ /** transitive comparison of 2 word objects. Used by operator<. @@ -163,14 +145,6 @@ public: } }; -struct WordComparer { - //! returns true if hypoA can be recombined with hypoB - bool operator()(const Word *a, const Word *b) const { - return *a < *b; - } -}; - - inline size_t hash_value(const Word& word) { return word.hash(); diff --git a/moses/WordLattice.cpp b/moses/WordLattice.cpp index 7804c9a58..c68d182b0 100644 --- a/moses/WordLattice.cpp +++ b/moses/WordLattice.cpp @@ -12,7 +12,7 @@ namespace Moses { WordLattice::WordLattice() : ConfusionNet() { - UTIL_THROW_IF2(&InputFeature::Instance() == NULL, + UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified"); } @@ -57,9 +57,9 @@ InitializeFromPCNDataType const std::string& debug_line) { // const StaticData &staticData = StaticData::Instance(); - const InputFeature &inputFeature = InputFeature::Instance(); - size_t numInputScores = inputFeature.GetNumInputScores(); - size_t numRealWordCount = inputFeature.GetNumRealWordsInInput(); + const InputFeature *inputFeature = InputFeature::InstancePtr(); + size_t numInputScores = inputFeature->GetNumInputScores(); + size_t numRealWordCount = inputFeature->GetNumRealWordsInInput(); size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength(); diff --git a/moses/WordsBitmap.cpp b/moses/WordsBitmap.cpp index 17340ffac..d332b1243 100644 --- a/moses/WordsBitmap.cpp +++ b/moses/WordsBitmap.cpp @@ -19,6 +19,7 @@ License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ +#include <boost/functional/hash.hpp> #include "WordsBitmap.h" namespace Moses @@ -34,4 +35,27 @@ bool WordsBitmap::IsAdjacent(size_t startPos, size_t endPos) const endPos == GetLastGapPos(); } +// for unordered_set in stack +size_t WordsBitmap::hash() const +{ + size_t ret = boost::hash_value(m_bitmap); + return ret; +} + +bool WordsBitmap::operator==(const WordsBitmap& other) const +{ + return m_bitmap == other.m_bitmap; +} + +// friend +std::ostream& operator<<(std::ostream& out, const WordsBitmap& wordsBitmap) +{ + for (size_t i = 0 ; i < wordsBitmap.m_bitmap.size() ; i++) { + out << int(wordsBitmap.GetValue(i)); + } + return out; } + +} // namespace + + diff --git a/moses/WordsBitmap.h b/moses/WordsBitmap.h index c1dcb8acf..3e2a9e38d 100644 --- a/moses/WordsBitmap.h +++ b/moses/WordsBitmap.h @@ -183,26 +183,6 @@ public: return m_bitmap.size(); } - //! transitive comparison of WordsBitmap - inline int Compare (const WordsBitmap &compare) const { - // -1 = less than - // +1 = more than - // 0 = same - - size_t thisSize = GetSize() - ,compareSize = compare.GetSize(); - - if (thisSize != compareSize) { - return (thisSize < compareSize) ? -1 : 1; - } - return std::memcmp( - &m_bitmap[0], &compare.m_bitmap[0], thisSize * sizeof(bool)); - } - - bool operator< (const WordsBitmap &compare) const { - return Compare(compare) < 0; - } - inline size_t GetEdgeToTheLeftOf(size_t l) const { if (l == 0) return l; while (l && !m_bitmap[l-1]) { @@ -261,17 +241,15 @@ public: return id + (1<<16) * start; } + // for unordered_set in stack + size_t hash() const; + bool operator==(const WordsBitmap& other) const; + bool operator!=(const WordsBitmap& other) const { + return !(*this == other); + } + TO_STRING(); }; -// friend -inline std::ostream& operator<<(std::ostream& out, const WordsBitmap& wordsBitmap) -{ - for (size_t i = 0 ; i < wordsBitmap.m_bitmap.size() ; i++) { - out << int(wordsBitmap.GetValue(i)); - } - return out; -} - } #endif diff --git a/moses/WordsRange.h b/moses/WordsRange.h index 4a38ecde7..98f96313f 100644 --- a/moses/WordsRange.h +++ b/moses/WordsRange.h @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_WordsRange_h #include <iostream> +#include <boost/functional/hash.hpp> #include "TypeDef.h" #include "Util.h" #include "util/exception.hh" @@ -94,6 +95,12 @@ public: TO_STRING(); }; +inline size_t hash_value(const WordsRange& range) +{ + size_t seed = range.GetStartPos(); + boost::hash_combine(seed, range.GetEndPos()); + return seed; +} } #endif diff --git a/moses/parameters/CubePruningOptions.cpp b/moses/parameters/CubePruningOptions.cpp index a8710c681..b4ebb0d5e 100644 --- a/moses/parameters/CubePruningOptions.cpp +++ b/moses/parameters/CubePruningOptions.cpp @@ -13,6 +13,7 @@ namespace Moses param.SetParameter(diversity, "cube-pruning-diversity", DEFAULT_CUBE_PRUNING_DIVERSITY); param.SetParameter(lazy_scoring, "cube-pruning-lazy-scoring", false); + param.SetParameter(deterministic_search, "cube-pruning-deterministic-search", false); return true; } @@ -30,20 +31,37 @@ namespace Moses if (si != params.end()) diversity = xmlrpc_c::value_int(si->second); si = params.find("cube-pruning-lazy-scoring"); - if (si != params.end()) - { - std::string spec = xmlrpc_c::value_string(si->second); - if (spec == "true" or spec == "on" or spec == "1") - lazy_scoring = true; - else if (spec == "false" or spec == "off" or spec == "0") - lazy_scoring = false; - else + if (si != params.end()) { - char const* msg - = "Error parsing specification for cube-pruning-lazy-scoring"; - xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE); + std::string spec = xmlrpc_c::value_string(si->second); + if (spec == "true" or spec == "on" or spec == "1") + lazy_scoring = true; + else if (spec == "false" or spec == "off" or spec == "0") + lazy_scoring = false; + else + { + char const* msg + = "Error parsing specification for cube-pruning-lazy-scoring"; + xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE); + } } - } + + si = params.find("cube-pruning-deterministic-search"); + if (si != params.end()) + { + std::string spec = xmlrpc_c::value_string(si->second); + if (spec == "true" or spec == "on" or spec == "1") + deterministic_search = true; + else if (spec == "false" or spec == "off" or spec == "0") + deterministic_search = false; + else + { + char const* msg + = "Error parsing specification for cube-pruning-deterministic-search"; + xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE); + } + } + return true; } #endif diff --git a/moses/parameters/CubePruningOptions.h b/moses/parameters/CubePruningOptions.h index 5d27be6a9..961b1a479 100644 --- a/moses/parameters/CubePruningOptions.h +++ b/moses/parameters/CubePruningOptions.h @@ -12,6 +12,7 @@ namespace Moses size_t pop_limit; size_t diversity; bool lazy_scoring; + bool deterministic_search; bool init(Parameter const& param); CubePruningOptions(Parameter const& param); diff --git a/moses/server/Session.h b/moses/server/Session.h index daba6d006..f56780839 100644 --- a/moses/server/Session.h +++ b/moses/server/Session.h @@ -17,7 +17,7 @@ namespace MosesServer{ time_t start_time; time_t last_access; boost::shared_ptr<Moses::ContextScope> const scope; // stores local info - + SPTR<std::map<std::string,float> > m_context_weights; Session(uint64_t const session_id) @@ -27,6 +27,8 @@ namespace MosesServer{ } bool is_new() const { return last_access == start_time; } + + void setup(std::map<std::string, xmlrpc_c::value> const& params); }; class SessionCache diff --git a/moses/server/TranslationRequest.cpp b/moses/server/TranslationRequest.cpp index bdccd8b87..8c1efb628 100644 --- a/moses/server/TranslationRequest.cpp +++ b/moses/server/TranslationRequest.cpp @@ -39,9 +39,11 @@ void TranslationRequest:: Run() { - std::map<std::string,xmlrpc_c::value>const& params = m_paramList.getStruct(0); + typedef std::map<std::string,xmlrpc_c::value> param_t; + param_t const& params = m_paramList.getStruct(0); parse_request(params); // cerr << "SESSION ID" << ret->m_session_id << endl; + if (m_session_id) { Session const& S = m_translator->get_session(m_session_id); @@ -51,6 +53,11 @@ Run() } else m_scope.reset(new Moses::ContextScope); + // settings within the session scope + param_t::const_iterator si = params.find("context-weights"); + if (si != params.end()) + m_context_weights = m_scope->GetContextWeights(&si->second); + Moses::StaticData const& SD = Moses::StaticData::Instance(); //Make sure alternative paths are retained, if necessary @@ -239,6 +246,7 @@ TranslationRequest(xmlrpc_c::paramList const& paramList, m_options = StaticData::Instance().options(); } + void TranslationRequest:: parse_request(std::map<std::string, xmlrpc_c::value> const& params) @@ -262,7 +270,7 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params) m_session_id = xmlrpc_c::value_int(si->second); else m_session_id = 0; - + m_withAlignInfo = check(params, "align"); m_withWordAlignInfo = check(params, "word-align"); m_withGraphInfo = check(params, "sg"); @@ -347,7 +355,8 @@ pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key, ostringstream target; BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) output_phrase(target, e->GetCurrTargetPhrase()); - std::cerr << "SERVER TRANSLATION: " << target.str() << std::endl; + XVERBOSE(1,"SERVER TRANSLATION: " << target.str() << std::endl); + dest[key] = xmlrpc_c::value_string(target.str()); if (m_withAlignInfo) { diff --git a/phrase-extract/extract-ghkm/Subgraph.cpp b/phrase-extract/extract-ghkm/Subgraph.cpp index ef4b07200..e2e2f6c99 100644 --- a/phrase-extract/extract-ghkm/Subgraph.cpp +++ b/phrase-extract/extract-ghkm/Subgraph.cpp @@ -137,6 +137,11 @@ void Subgraph::RecursivelyPrintTree(const Node *n, std::ostream &out) const for (std::vector<Node *>::const_iterator p(children.begin()); p != children.end(); ++p) { Node *child = *p; + if (child->GetType() == SOURCE) { + // This is possible due to the heuristic for attaching unaligned + // source words. + continue; + } out << " "; RecursivelyPrintTree(child,out); } diff --git a/scripts/OSM/OSM-Train.perl b/scripts/OSM/OSM-Train.perl index cf3bccff2..272bbaa5b 100755 --- a/scripts/OSM/OSM-Train.perl +++ b/scripts/OSM/OSM-Train.perl @@ -14,7 +14,7 @@ print STDERR "Training OSM - Start\n".`date`; my $ORDER = 5; my $OUT_DIR = "/tmp/osm.$$"; my $___FACTOR_DELIMITER = "|"; -my ($MOSES_SRC_DIR,$CORPUS_F,$CORPUS_E,$ALIGNMENT,$SRILM_DIR,$FACTOR,$LMPLZ); +my ($MOSES_SRC_DIR,$CORPUS_F,$CORPUS_E,$ALIGNMENT,$SRILM_DIR,$FACTOR,$LMPLZ,$DOMAIN,$TUNE,$INP_EXT,$OP_EXT); my $cmd; @@ -29,6 +29,10 @@ die("ERROR: wrong syntax when invoking OSM-Train.perl") 'alignment=s' => \$ALIGNMENT, 'order=i' => \$ORDER, 'factor=s' => \$FACTOR, + 'input-extension=s' => \$INP_EXT, + 'output-extension=s' => \$OP_EXT, + 'tune=s' => \$TUNE, + 'domain=s' => \$DOMAIN, 'srilm-dir=s' => \$SRILM_DIR, 'lmplz=s' => \$LMPLZ, 'out-dir=s' => \$OUT_DIR); @@ -74,19 +78,172 @@ if (defined($FACTOR)) { `ln -s $corpus_stem_f.$factor_val.$ext_f $OUT_DIR/$factor_val/f`; `ln -s $corpus_stem_e.$factor_val.$ext_e $OUT_DIR/$factor_val/e`; - create_model($factor_val); + + if (defined($TUNE) && defined($DOMAIN) && $factor_val eq "0-0") + { + + die("ERROR: For Interpolated OSM model, you need SRILM") + unless -e $SRILM_DIR; + + `mkdir $OUT_DIR/TUNE`; + + `$MOSES_SRC_DIR/scripts/training/reduce-factors.perl --corpus $TUNE.$INP_EXT --reduced $OUT_DIR/TUNE/tune.$INP_EXT --factor 0`; + `$MOSES_SRC_DIR/scripts/training/reduce-factors.perl --corpus $TUNE.$OP_EXT --reduced $OUT_DIR/TUNE/tune.$OP_EXT --factor 0`; + + create_interpolated_model($factor_val); + } + else + { + create_model($factor_val); + } } } else { `ln -s $CORPUS_F $OUT_DIR/f`; `ln -s $CORPUS_E $OUT_DIR/e`; - create_model(""); + + + + if (defined($TUNE) && defined($DOMAIN)) + { + + die("ERROR: For Interpolated OSM model, you need SRILM") + unless -e $SRILM_DIR; + + `mkdir $OUT_DIR/TUNE`; + + `cp $TUNE.$INP_EXT --reduced $OUT_DIR/TUNE/tune.$INP_EXT`; + `cp $TUNE.$OP_EXT --reduced $OUT_DIR/TUNE/tune.$OP_EXT`; + + create_interpolated_model(""); + } + else + { + create_model(""); + } + } # create model print "Training OSM - End".`date`; + +sub read_domain_file{ + + open(my $fh, '<:encoding(UTF-8)', $DOMAIN) + or die "Could not open file '$DOMAIN' $!"; + + my @corpora; + + while (my $row = <$fh>) { + chomp $row; + + my ($num,$dom) = split(/\ /,$row); + + push @corpora, $dom; + push @corpora, $num; + + #print "$dom $num\n"; + } + + return @corpora; + +} + +sub create_interpolated_model{ + + + my ($factor_val) = @_; + my $fNum = 0; + my $dName; + my @corpora = read_domain_file(); + my $i = 0; + + while($i < scalar(@corpora)) + { + $dName = "$OUT_DIR/$factor_val/$corpora[$i]"; + $cmd = "mkdir $dName"; + `$cmd`; + + my $cal = $corpora[$i+1] - $fNum; + $cmd = "head -$corpora[$i+1] $OUT_DIR/$factor_val/e | tail -$cal > $dName/e"; + `$cmd`; + $cmd = "head -$corpora[$i+1] $OUT_DIR/$factor_val/f | tail -$cal > $dName/f"; + `$cmd`; + $cmd = "head -$corpora[$i+1] $OUT_DIR/align | tail -$cal > $dName/align"; + `$cmd`; + + #print STDERR "Flip Alignment\n"; + #`$MOSES_SRC_DIR/scripts/OSM/flipAlignment.perl $dName/alignment > $dName/align`; + + print STDERR "Extracting Singletons\n"; + $cmd = "$MOSES_SRC_DIR/scripts/OSM/extract-singletons.perl $dName/e $dName/f $dName/align > $dName/Singletons"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + + print STDERR "Converting Bilingual Sentence Pair into Operation Corpus\n"; + $cmd = "$MOSES_SRC_DIR/bin/generateSequences $dName/e $dName/f $dName/align $dName/Singletons > $dName/opCorpus"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + + print STDERR "Learning Operation Sequence Translation Model\n"; + if (defined($SRILM_DIR)) { + $cmd = "$SRILM_DIR/ngram-count -kndiscount -order $ORDER -unk -text $dName/opCorpus -lm $dName/operationLM 2>> /dev/stderr"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + } + else { + $cmd = "$LMPLZ -T $OUT_DIR --order $ORDER --text $dName/opCorpus --arpa $dName/operationLM --prune 0 0 1 2>> /dev/stderr"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + } + + print "$cmd\n"; + $fNum = $corpora[$i+1]; + $i = $i+2; + } + + + `$MOSES_SRC_DIR/scripts/OSM/flipAlignment.perl $TUNE.align > $OUT_DIR/TUNE/tune.align`; + + print STDERR "Extracting Singletons\n"; + $cmd = "$MOSES_SRC_DIR/scripts/OSM/extract-singletons.perl $OUT_DIR/TUNE/tune.$OP_EXT $OUT_DIR/TUNE/tune.$INP_EXT $OUT_DIR/TUNE/tune.align > $OUT_DIR/TUNE/Singletons"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + + print STDERR "Converting Bilingual Sentence Pair into Operation Corpus\n"; + $cmd = "$MOSES_SRC_DIR/bin/generateSequences $OUT_DIR/TUNE/tune.$OP_EXT $OUT_DIR/TUNE/tune.$INP_EXT $OUT_DIR/TUNE/tune.align $OUT_DIR/TUNE/Singletons > $OUT_DIR/TUNE/tune.opCorpus"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + + + print STDERR "Interpolating OSM Models\n"; + $cmd = "$MOSES_SRC_DIR/scripts/ems/support/interpolate-lm.perl --tuning $OUT_DIR/TUNE/tune.opCorpus --name $OUT_DIR/$factor_val/operationLM --srilm $SRILM_DIR --lm "; + + $i = 0; + $dName = "$OUT_DIR/$factor_val/$corpora[$i]/operationLM"; + $cmd = $cmd . $dName; + $i = $i+2; + + while($i < scalar(@corpora)) + { + $cmd = $cmd . ","; + $dName = "$OUT_DIR/$factor_val/$corpora[$i]/operationLM"; + $cmd = $cmd . $dName; + $i = $i+2; + } + + print STDERR "Executing: $cmd\n"; + `$cmd`; + + print STDERR "Binarizing\n"; + $cmd = "$MOSES_SRC_DIR/bin/build_binary $OUT_DIR/$factor_val/operationLM $OUT_DIR/$factor_val/operationLM.bin"; + print STDERR "Executing: $cmd\n"; + system($cmd) == 0 or die("system $cmd failed: $?"); + +} + sub create_model{ my ($factor_val) = @_; diff --git a/scripts/ems/example/config.basic b/scripts/ems/example/config.basic index 60a4f1a84..006d7022e 100644 --- a/scripts/ems/example/config.basic +++ b/scripts/ems/example/config.basic @@ -391,6 +391,28 @@ alignment-symmetrization-method = grow-diag-final-and #operation-sequence-model-order = 5 #operation-sequence-model-settings = "-lmplz '$moses-src-dir/bin/lmplz -S 40% -T $working-dir/model/tmp'" # +# OR if you want to use with SRILM +# +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64" + +## Class-based Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Koehn, Schmid, Fraser (COLING, 2014). +#Investigating the Usefulness of Generalized Word Representations in SMT +# +#operation-sequence-model-settings = "--factor 0-0+1-1" + +## Interpolated Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Sajjad, Joty, Abdelali and Vogel (Mt Summit, 2015). +# Using Joint Models for Domain Adaptation in Statistical Machine Translation +# +#interpolated-operation-sequence-model = "yes" +#operation-sequence-model-order = 5 +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64 --tune /path-to-tune-folder/tune_file" +#Interpolated OSM can only be used with SRILM because of the interpolation script + + # if OSM training should be skipped, point to OSM Model #osm-model = diff --git a/scripts/ems/example/config.factored b/scripts/ems/example/config.factored index dd4227ae9..ef5b81010 100644 --- a/scripts/ems/example/config.factored +++ b/scripts/ems/example/config.factored @@ -411,9 +411,30 @@ alignment-symmetrization-method = grow-diag-final-and #operation-sequence-model-order = 5 #operation-sequence-model-settings = "-lmplz '$moses-src-dir/bin/lmplz -S 40% -T $working-dir/model/tmp'" # +# OR if you want to use with SRILM +# +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64" + +## Class-based Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Koehn, Schmid, Fraser (COLING, 2014). +#Investigating the Usefulness of Generalized Word Representations in SMT +# +#operation-sequence-model-settings = "--factor 0-0+1-1" + +## Interpolated Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Sajjad, Joty, Abdelali and Vogel (Mt Summit, 2015). +# Using Joint Models for Domain Adaptation in Statistical Machine Translation +# +#interpolated-operation-sequence-model = "yes" +#operation-sequence-model-order = 5 +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64 --tune /path-to-tune-folder/tune_file" +#Interpolated OSM can only be used with SRILM because of the interpolation script + + # if OSM training should be skipped, point to OSM Model #osm-model = - ### unsupervised transliteration module # Durrani, Sajjad, Hoang and Koehn (EACL, 2014). # "Integrating an Unsupervised Transliteration Model diff --git a/scripts/ems/example/config.toy b/scripts/ems/example/config.toy index 0274a7fc0..0b2975b22 100644 --- a/scripts/ems/example/config.toy +++ b/scripts/ems/example/config.toy @@ -373,8 +373,30 @@ alignment-symmetrization-method = grow-diag-final-and # #operation-sequence-model = "yes" #operation-sequence-model-order = 5 -#operation-sequence-model-settings = "" +#operation-sequence-model-settings = "-lmplz '$moses-src-dir/bin/lmplz -S 40% -T $working-dir/model/tmp'" # +# OR if you want to use with SRILM +# +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64" + +## Class-based Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Koehn, Schmid, Fraser (COLING, 2014). +#Investigating the Usefulness of Generalized Word Representations in SMT +# +#operation-sequence-model-settings = "--factor 0-0+1-1" + +## Interpolated Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Sajjad, Joty, Abdelali and Vogel (Mt Summit, 2015). +# Using Joint Models for Domain Adaptation in Statistical Machine Translation +# +#interpolated-operation-sequence-model = "yes" +#operation-sequence-model-order = 5 +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64 --tune /path-to-tune-folder/tune_file" +#Interpolated OSM can only be used with SRILM because of the interpolation script + + # if OSM training should be skipped, point to OSM Model #osm-model = diff --git a/scripts/ems/example/config.toy.bilinguallm b/scripts/ems/example/config.toy.bilinguallm index 7fda90933..abda2adc2 100644 --- a/scripts/ems/example/config.toy.bilinguallm +++ b/scripts/ems/example/config.toy.bilinguallm @@ -389,8 +389,30 @@ alignment-symmetrization-method = grow-diag-final-and # #operation-sequence-model = "yes" #operation-sequence-model-order = 5 -#operation-sequence-model-settings = "" +#operation-sequence-model-settings = "-lmplz '$moses-src-dir/bin/lmplz -S 40% -T $working-dir/model/tmp'" # +# OR if you want to use with SRILM +# +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64" + +## Class-based Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Koehn, Schmid, Fraser (COLING, 2014). +#Investigating the Usefulness of Generalized Word Representations in SMT +# +#operation-sequence-model-settings = "--factor 0-0+1-1" + +## Interpolated Operation Sequence Model (OSM) +# if OSM has to be enabled with factors then add factors as below. +# Durrani, Sajjad, Joty, Abdelali and Vogel (Mt Summit, 2015). +# Using Joint Models for Domain Adaptation in Statistical Machine Translation +# +#interpolated-operation-sequence-model = "yes" +#operation-sequence-model-order = 5 +#operation-sequence-model-settings = "--srilm-dir /path-to-srilm/bin/i686-m64 --tune /path-to-tune-folder/tune_file" +#Interpolated OSM can only be used with SRILM because of the interpolation script + + # if OSM training should be skipped, point to OSM Model #osm-model = diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index 7de32ad99..562cf8ec2 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -533,7 +533,7 @@ build-domains in: CORPUS:post-split-factorized-stem out: domains default-name: model/domains - ignore-unless: domain-features mml-filter-corpora + ignore-unless: domain-features mml-filter-corpora interpolated-operation-sequence-model template: $moses-script-dir/ems/support/build-domain-file-from-subcorpora.perl $input-extension IN > OUT final-model: yes mml-score @@ -702,7 +702,14 @@ build-osm out: osm-model ignore-unless: operation-sequence-model rerun-on-change: operation-sequence-model training-options script giza-settings operation-sequence-model-settings - template: $moses-script-dir/OSM/OSM-Train.perl --corpus-f IN0.$input-extension --corpus-e IN0.$output-extension --alignment IN1.$alignment-symmetrization-method --order $operation-sequence-model-order --out-dir OUT --moses-src-dir $moses-src-dir $operation-sequence-model-settings + template: $moses-script-dir/OSM/OSM-Train.perl --corpus-f IN0.$input-extension --corpus-e IN0.$output-extension --alignment IN1.$alignment-symmetrization-method --order $operation-sequence-model-order --out-dir OUT --moses-src-dir $moses-src-dir --input-extension $input-extension --output-extension $output-extension $operation-sequence-model-settings + default-name: model/OSM +build-interpolated-osm + in: corpus word-alignment domains + out: osm-model + ignore-unless: interpolated-operation-sequence-model + rerun-on-change: interpolated-operation-sequence-model training-options script giza-settings operation-sequence-model-settings + template: $moses-script-dir/OSM/OSM-Train.perl --corpus-f IN0.$input-extension --corpus-e IN0.$output-extension --alignment IN1.$alignment-symmetrization-method --order $operation-sequence-model-order --out-dir OUT --moses-src-dir $moses-src-dir --input-extension $input-extension --output-extension $output-extension $operation-sequence-model-settings --domain IN2 default-name: model/OSM build-transliteration-model in: corpus word-alignment @@ -940,6 +947,21 @@ parse-input-devtest pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval ignore-unless: use-mira template: $input-parser < IN > OUT +parse-relax-input + in: split-input + out: input + default-name: tuning/input.parse-relaxed + pass-unless: input-parse-relaxer + pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval + template: $input-parse-relaxer < IN > OUT +parse-relax-input-devtest + in: split-input-devtest + out: input-devtest + default-name: tuning/input.devtest.parse-relaxed + pass-unless: input-parse-relaxer + pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval + ignore-unless: use-mira + template: $input-parse-relaxer < IN > OUT factorize-input in: parsed-input out: factorized-input @@ -1001,35 +1023,20 @@ truecase-input-devtest ignore-unless: AND input-truecaser use-mira template: $input-truecaser -model IN1.$input-extension < IN > OUT split-input - in: truecased-input + in: truecased-input SPLITTER:splitter-model out: split-input - rerun-on-change: input-splitter SPLITTER:splitter-model + rerun-on-change: input-splitter default-name: tuning/input.split pass-unless: input-splitter - template: $input-splitter -model SPLITTER:splitter-model.$input-extension < IN > OUT + template: $input-splitter -model IN1.$input-extension < IN > OUT split-input-devtest - in: truecased-input-devtest + in: truecased-input-devtest SPLITTER:splitter-model out: split-input-devtest rerun-on-change: input-splitter default-name: tuning/input.devtest.split pass-unless: input-splitter ignore-unless: use-mira - template: $input-splitter -model SPLITTER:splitter-model.$input-extension < IN > OUT -parse-relax-input - in: split-input - out: input - default-name: tuning/input.parse-relaxed - pass-unless: input-parse-relaxer - pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval - template: $input-parse-relaxer < IN > OUT -parse-relax-input-devtest - in: split-input-devtest - out: input-devtest - default-name: tuning/input.devtest.parse-relaxed - pass-unless: input-parse-relaxer - pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval - ignore-unless: use-mira - template: $input-parse-relaxer < IN > OUT + template: $input-splitter -model IN1.$input-extension < IN > OUT reference-from-sgm in: reference-sgm input-sgm out: raw-reference @@ -1269,12 +1276,11 @@ truecase-input ignore-unless: input-truecaser template: $input-truecaser -model IN1.$input-extension < IN > OUT split-input - in: truecased-input + in: truecased-input SPLITTER:splitter-model out: split-input - rerun-on-change: input-splitter SPLITTER:splitter-model default-name: evaluation/input.split pass-unless: input-splitter - template: $input-splitter -model SPLITTER:splitter-model.$input-extension < IN > OUT + template: $input-splitter -model IN1.$input-extension < IN > OUT filter in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains TRAINING:transliteration-table out: filtered-dir diff --git a/scripts/ems/support/cache-model.perl b/scripts/ems/support/cache-model.perl index f4c7e9ef6..9d71d9072 100755 --- a/scripts/ems/support/cache-model.perl +++ b/scripts/ems/support/cache-model.perl @@ -8,9 +8,15 @@ use strict; +my $CAT_MODELS = 0; + die("ERROR: syntax is cache-model.perl moses.ini cache-dir") - unless scalar @ARGV == 2; -my ($CONFIG,$CACHE_DIR) = @ARGV; + unless scalar @ARGV >= 2; +my $CONFIG = $ARGV[0]; +my $CACHE_DIR = $ARGV[1]; +if (scalar(@ARGV) == 3) { + $CAT_MODELS = $ARGV[2]; +} # create dir (if nor already there) `mkdir -p $CACHE_DIR`; @@ -39,25 +45,25 @@ while(<OLD>) { my ($pre,$path,$post) = ($1,$2,$3); my $new_path; if (/^PhraseDictionaryCompact/) { - $new_path = &cache_file($path,".minphr"); + $new_path = &cache_file($path,".minphr", $CAT_MODELS); } elsif (/^PhraseDictionaryBinary/) { foreach my $suffix (".binphr.idx",".binphr.srctree.wa",".binphr.srcvoc",".binphr.tgtdata.wa",".binphr.tgtvoc") { - $new_path = &cache_file($path,$suffix); + $new_path = &cache_file($path,$suffix, $CAT_MODELS); } } elsif (/^LexicalReordering/ && -e "$path.minlexr") { - $new_path = &cache_file($path,".minlexr"); + $new_path = &cache_file($path,".minlexr", $CAT_MODELS); } elsif (/^LexicalReordering/ && -e "$path.binlexr.idx") { foreach my $suffix (".binlexr.idx",".binlexr.srctree",".binlexr.tgtdata",".binlexr.voc0",".binlexr.voc1") { - $new_path = &cache_file($path,$suffix); + $new_path = &cache_file($path,$suffix, $CAT_MODELS); } } # some other files may need some more special handling # but this works for me right now. feel free to add else { - $new_path = &cache_file($path,""); + $new_path = &cache_file($path,"", $CAT_MODELS); } print NEW "$pre$new_path$post\n" unless $just_update_timestamps; } @@ -72,7 +78,7 @@ close(OLD); print "$cached_config\n"; sub cache_file { - my ($path,$suffix) = @_; + my ($path,$suffix, $catModels) = @_; # add gzipped extension if that's what it is if (! -e "$path$suffix" && -e "$path$suffix.gz") { @@ -102,14 +108,19 @@ sub cache_file { # done if already there if (-e "$cached_path$suffix") { `touch $cached_path$suffix`; # update time stamp - return $cached_path; + } + else { + # okay, go for it + `touch $cached_path$suffix.lock`; + `cp $path$suffix $cached_path$suffix`; + `rm $cached_path$suffix.lock`; } - # okay, go for it - `touch $cached_path$suffix.lock`; - `cp $path$suffix $cached_path$suffix`; - `rm $cached_path$suffix.lock`; - + if ($catModels) { + my $cmd = "cat $cached_path* > /dev/null"; + print STDERR "Executing: $cmd\n"; + `$cmd`; + } return $cached_path; } diff --git a/scripts/generic/extract-parallel.perl b/scripts/generic/extract-parallel.perl index 2424c1bd2..87b6a8deb 100755 --- a/scripts/generic/extract-parallel.perl +++ b/scripts/generic/extract-parallel.perl @@ -44,7 +44,7 @@ my $phraseOrientationPriorsFile; my $splitCmdOption = ""; my $GZIP_EXEC; -if(`which pigz`) { +if(`which pigz 2> /dev/null`) { $GZIP_EXEC = 'pigz'; } else { @@ -374,7 +374,7 @@ sub NumStr($) sub GetSplitVersion($) { my $splitCmd = shift; - my $retVal = system("$splitCmd --help"); + my $retVal = system("$splitCmd --help > /dev/null"); if ($retVal != 0) { return 1; } diff --git a/scripts/generic/multi_moses.py b/scripts/generic/multi_moses.py new file mode 100755 index 000000000..01a38d8e6 --- /dev/null +++ b/scripts/generic/multi_moses.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python + +# Written by Michael Denkowski +# +# This file is part of moses. Its use is licensed under the GNU Lesser General +# Public License version 2.1 or, at your option, any later version. + +'''Parallelize decoding with multiple instances of moses on a local machine + +To use with mert-moses.pl, activate --multi-moses and set the number of moses +instances and threads per instance with --decoder-flags='--threads P:T:E' + +This script runs a specified number of moses instances, each using one or more +threads. The highest speed is generally seen with many single-threaded +instances while the lowest memory usage is seen with a single many-threaded +instance. It is recommended to use the maximum number of instances that will +fit into memory (up to the number of available CPUs) and distribute CPUs across +them equally. For example, a machine with 32 CPUs that can fit 3 copies of +moses into memory would use --threads 2:11:10 for 2 instances with 11 threads +each and an extra instance with 10 threads (3 instances total using all CPUs). + +Memory mapped models can be shared by multiple processes and increase the number +of instances that can fit into memory: + +Mmaped phrase tables (Ulrich Germann) +http://www.statmt.org/moses/?n=Advanced.Incremental#ntoc3 + +Mmaped mapped language models (Kenneth Heafield) +http://www.statmt.org/moses/?n=FactoredTraining.BuildingLanguageModel#ntoc19 +''' + +import collections +import os +import Queue +import signal +import subprocess +import sys +import threading + +HELP = '''Multiple process decoding with Moses + +Usage: + {} moses --config moses.ini [options] [decoder flags] + +Options: + --threads P:T:E + P: Number of parallel instances to run + T: Number of threads per instance + E: Number of threads in optional extra instance + (default 1:1:0, overrides [threads] in moses.ini. Specifying T + and E is optional, e.g. --threads 16 starts 16 single-threaded + instances) + --n-best-list nbest.out N [distinct]: location and size of N-best list + --show-weights: for mert-moses.pl, just call moses and exit + +Other options (decoder flags) are passed through to moses instances +''' + +# Defaults +INPUT = sys.stdin +PROCS = 1 +THREADS = 1 +EXTRA = 0 +DONE = threading.Event() +PID = os.getpid() +# A very long time, used as Queue operation timeout even though we don't +# actually want a timeout but we do want interruptibility +# (https://bugs.python.org/issue1360) +NEVER = 60 * 60 * 24 * 365 * 1000 + +# Single unit of computation: decode a line, output result, signal done +Task = collections.namedtuple('Task', ['id', 'line', 'out', 'event']) + + +def kill_main(msg): + '''kill -9 the main thread to stop everything immediately''' + sys.stderr.write('{}\n'.format(msg)) + os.kill(PID, signal.SIGKILL) + + +def gzopen(f): + '''Open plain or gzipped text''' + return gzip.open(f, 'rb') if f.endswith('.gz') else open(f, 'r') + + +def run_instance(cmd_base, threads, tasks, n_best=False): + '''Run an instance of moses that processes tasks (input lines) from a + queue using a specified number of threads''' + cmd = cmd_base[:] + cmd.append('--threads') + cmd.append(str(threads)) + try: + # Queue of tasks instance is currently working on, limited to the number of + # threads. The queue should be kept full for optimal CPU usage. + work = Queue.Queue(maxsize=threads) + # Multi-threaded instance + moses = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Read and handle instance output as available + def handle_output(): + while True: + # Output line triggers task completion + line = moses.stdout.readline() + # End of output (instance finished) + if not line: + break + task = work.get(timeout=NEVER) + if n_best: + # Read and copy lines until sentinel line, copy real line id + # id ||| hypothesis words ||| feature scores ||| total score + (first_i, rest) = line.split(' ||| ', 1) + task.out.append(' ||| '.join((task.id, rest))) + while True: + line = moses.stdout.readline() + (i, rest) = line.split(' ||| ', 1) + # Sentinel + if i != first_i: + break + task.out.append(' ||| '.join((task.id, rest))) + else: + task.out.append(line) + # Signal task done + task.event.set() + # Output thread + handler = threading.Thread(target=handle_output, args=()) + # Daemon: guaranteed to finish before non-daemons + handler.setDaemon(True) + handler.start() + + # Input thread: take tasks as they are available and add them to work + # queue. Stop when DONE encountered. + while True: + task = tasks.get(timeout=NEVER) + work.put(task, timeout=NEVER) + if task.event == DONE: + break + if n_best: + # Input line followed by blank line (sentinel) + moses.stdin.write(task.line) + moses.stdin.write('\n') + else: + moses.stdin.write(task.line) + + # Cleanup + moses.stdin.close() + moses.wait() + handler.join() + + except: + kill_main('Error with moses instance: see stderr') + + +def write_results(results, n_best=False, n_best_out=None): + '''Write out results (output lines) from a queue as they are populated''' + while True: + task = results.get(timeout=NEVER) + if task.event == DONE: + break + task.event.wait() + if n_best: + # Write top-best and N-best + # id ||| hypothesis words ||| feature scores ||| total score + top_best = task.out[0].split(' ||| ', 2)[1] + # Except don't write top-best if writing N-best to stdout "-" + if n_best_out != sys.stdout: + sys.stdout.write('{}\n'.format(top_best)) + sys.stdout.flush() + for line in task.out: + n_best_out.write(line) + n_best_out.flush() + else: + sys.stdout.write(task.out[0]) + sys.stdout.flush() + + +def main(argv): + # Defaults + moses_ini = None + input = INPUT + procs = PROCS + threads = THREADS + extra = EXTRA + n_best = False + n_best_file = None + n_best_size = None + n_best_distinct = False + n_best_out = None + show_weights = False + + # Decoder command + cmd = argv[1:] + + # Parse special options and remove from cmd + i = 1 + while i < len(cmd): + if cmd[i] in ('-f', '-config', '--config'): + moses_ini = cmd[i + 1] + # Do not remove from cmd + i += 2 + elif cmd[i] in ('-i', '-input-file', '--input-file'): + input = gzopen(cmd[i + 1]) + cmd = cmd[:i] + cmd[i + 2:] + elif cmd[i] in ('-th', '-threads', '--threads'): + # P:T:E + args = cmd[i + 1].split(':') + procs = int(args[0]) + if len(args) > 1: + threads = int(args[1]) + if len(args) > 2: + extra = int(args[2]) + cmd = cmd[:i] + cmd[i + 2:] + elif cmd[i] in ('-n-best-list', '--n-best-list'): + n_best = True + n_best_file = cmd[i + 1] + n_best_size = cmd[i + 2] + # Optional "distinct" + if i + 3 < len(cmd) and cmd[i + 3] == 'distinct': + n_best_distinct = True + cmd = cmd[:i] + cmd[i + 4:] + else: + cmd = cmd[:i] + cmd[i + 3:] + # Handled specially for mert-moses.pl + elif cmd[i] in ('-show-weights', '--show-weights'): + show_weights = True + # Do not remove from cmd + i += 1 + else: + i += 1 + + # If mert-moses.pl passes -show-weights, just call moses + if show_weights: + sys.stdout.write(subprocess.check_output(cmd)) + sys.stdout.flush() + sys.exit(0) + + # Check inputs + if not (len(cmd) > 0 and moses_ini): + sys.stderr.write(HELP.format(os.path.basename(argv[0]))) + sys.exit(2) + if not (os.path.isfile(cmd[0]) and os.access(cmd[0], os.X_OK)): + raise Exception('moses "{}" is not executable\n'.format(cmd[0])) + + # Report settings + sys.stderr.write('Moses flags: {}\n'.format(' '.join('\'{}\''.format(s) if ' ' in s else s for s in cmd[1:]))) + sys.stderr.write('Instances: {}\n'.format(procs)) + sys.stderr.write('Threads per: {}\n'.format(threads)) + if extra: + sys.stderr.write('Extra: {}\n'.format(extra)) + if n_best: + sys.stderr.write('N-best list: {} ({}{})\n'.format(n_best_file, n_best_size, ', distinct' if n_best_distinct else '')) + + # Task and result queues (buffer 8 * total threads input lines) + tasks = Queue.Queue(maxsize=(8 * ((procs * threads) + extra))) + results = Queue.Queue() + + # N-best capture + if n_best: + cmd.append('--n-best-list') + cmd.append('-') + cmd.append(n_best_size) + if n_best_distinct: + cmd.append('distinct') + if n_best_file == '-': + n_best_out = sys.stdout + else: + n_best_out = open(n_best_file, 'w') + + # Start instances + instances = [] + for i in range(procs + (1 if extra else 0)): + t = threading.Thread(target=run_instance, args=(cmd, (threads if i < procs else extra), tasks, n_best)) + instances.append(t) + # Daemon: guaranteed to finish before non-daemons + t.setDaemon(True) + t.start() + + # Start results writer + writer = threading.Thread(target=write_results, args=(results, n_best, n_best_out)) + writer.start() + + # Main loop: queue task for each input line + id = 0 + while True: + line = input.readline() + if not line: + break + # (input, out lines, err lines, "done" event) + task = Task(str(id), line, [], threading.Event()) + results.put(task, timeout=NEVER) + tasks.put(task, timeout=NEVER) + id += 1 + + # Tell instances to exit + for t in instances: + tasks.put(Task(None, None, None, DONE), timeout=NEVER) + for t in instances: + t.join() + + # Stop results writer + results.put(Task(None, None, None, DONE), timeout=NEVER) + writer.join() + + # Cleanup + if n_best: + n_best_out.close() + + +if __name__ == '__main__': + try: + main(sys.argv) + except: + kill_main('Error with main I/O: see stderr') diff --git a/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga b/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga new file mode 100644 index 000000000..d6c946946 --- /dev/null +++ b/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga @@ -0,0 +1,48 @@ + +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +Á +É +Í +Ó +Ú + +Uacht +Dr +B.Arch + +m.sh +.i +Co +Cf +cf +i.e +r +Chr +lch #NUMERIC_ONLY# +lgh #NUMERIC_ONLY# +uimh #NUMERIC_ONLY# diff --git a/scripts/tokenizer/tokenizer.perl b/scripts/tokenizer/tokenizer.perl index e08bac941..a1eb01c0f 100755 --- a/scripts/tokenizer/tokenizer.perl +++ b/scripts/tokenizer/tokenizer.perl @@ -305,7 +305,7 @@ sub tokenize #special case for "1990's" $text =~ s/([\p{IsN}])[']([s])/$1 '$2/g; } - elsif (($language eq "fr") or ($language eq "it")) + elsif (($language eq "fr") or ($language eq "it") or ($language eq "ga")) { #split contractions left $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; diff --git a/scripts/tokenizer/tokenizer_PTB.perl b/scripts/tokenizer/tokenizer_PTB.perl index 46b14775c..1d44b30be 100755 --- a/scripts/tokenizer/tokenizer_PTB.perl +++ b/scripts/tokenizer/tokenizer_PTB.perl @@ -298,7 +298,7 @@ sub tokenize $text =~ s/([Dd]) 'ye/$1' ye/g; } - elsif (($language eq "fr") or ($language eq "it")) + elsif (($language eq "fr") or ($language eq "it") or ($language eq "ga")) { #split contractions left $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; diff --git a/scripts/training/bilingual-lm/train_nplm.py b/scripts/training/bilingual-lm/train_nplm.py index 3280d506e..7fe8c9838 100755 --- a/scripts/training/bilingual-lm/train_nplm.py +++ b/scripts/training/bilingual-lm/train_nplm.py @@ -52,9 +52,9 @@ parser.set_defaults( ngram_size=14, minibatch_size=1000, noise=100, - hidden=750, + hidden=0, input_embedding=150, - output_embedding=150, + output_embedding=750, threads=1, output_model="train.10k", output_dir=None, diff --git a/scripts/training/mert-moses.pl b/scripts/training/mert-moses.pl index 8b71f92b0..6c08e6374 100755 --- a/scripts/training/mert-moses.pl +++ b/scripts/training/mert-moses.pl @@ -166,6 +166,10 @@ my $prev_aggregate_nbl_size = -1; # number of previous step to consider when loa # and so on my $maximum_iterations = 25; +# Multiple instance parallelization +my $___MULTI_MOSES = "$SCRIPTS_ROOTDIR/generic/multi_moses.py"; +my $___USE_MULTI_MOSES = undef; + # Simulated post-editing my $___MOSES_SIM_PE = "$SCRIPTS_ROOTDIR/generic/moses_sim_pe.py"; my $___DEV_SYMAL = undef; @@ -227,7 +231,8 @@ GetOptions( "promix-training=s" => \$__PROMIX_TRAINING, "promix-table=s" => \@__PROMIX_TABLES, "threads=i" => \$__THREADS, - "spe-symal=s" => \$___DEV_SYMAL + "spe-symal=s" => \$___DEV_SYMAL, + "multi-moses" => \$___USE_MULTI_MOSES ) or exit(1); # the 4 required parameters can be supplied on the command line directly @@ -325,6 +330,9 @@ Options: (parameter sets factor [0;1] given to current weights) --spe-symal=SYMAL ... Use simulated post-editing when decoding. (SYMAL aligns input to refs) + --multi-moses ... Use multiple instances of moses instead of threads for decoding + (Use with --decoder-flags='-threads N' to get N instances, each of + which uses a single thread (overrides threads in moses.ini)) "; exit 1; } @@ -1305,6 +1313,10 @@ sub run_decoder { $decoder_cmd = "$___DECODER $___DECODER_FLAGS -config $___CONFIG"; $decoder_cmd .= " -inputtype $___INPUTTYPE" if defined($___INPUTTYPE); $decoder_cmd .= " $decoder_config $lsamp_cmd $nbest_list_cmd -input-file $___DEV_F"; + if (defined $___USE_MULTI_MOSES) { + # If requested, prefix full decoder command with multi-moses wrapper + $decoder_cmd = "$___MULTI_MOSES $decoder_cmd"; + } if (defined $___DEV_SYMAL) { # If simulating post-editing, route command through moses_sim_pe.py # Always use single (first) reference. Simulated post-editing undefined for multiple references. diff --git a/scripts/training/train-model.perl b/scripts/training/train-model.perl index 56cbf016c..d87099e7e 100755 --- a/scripts/training/train-model.perl +++ b/scripts/training/train-model.perl @@ -415,7 +415,7 @@ else { } my $GZIP_EXEC; -if(`which pigz`) { +if(`which pigz 2> /dev/null`) { $GZIP_EXEC = 'pigz'; } else { diff --git a/scripts/training/train-neurallm.py b/scripts/training/train-neurallm.py index d86286f67..f8ef17ff9 100755 --- a/scripts/training/train-neurallm.py +++ b/scripts/training/train-neurallm.py @@ -92,6 +92,15 @@ parser.add_argument( parser.add_argument( "--mmap", dest="mmap", action="store_true", help="Use memory-mapped file (for lower memory consumption).") +parser.add_argument( + "--dropout", dest="dropout", action="store", + help="Pass dropout to nplm") +parser.add_argument( + "--input-dropout", dest="input_dropout", action="store", + help="Pass input dropout to nplm") +parser.add_argument( + "--extra-settings", dest="extra_settings", + help="Extra settings for nplm") parser.set_defaults( working_dir="working", @@ -129,6 +138,7 @@ def main(options): os.makedirs(options.output_dir) numberized_file = os.path.basename(options.corpus_stem) + '.numberized' + vocab_file =os.path.join(options.working_dir, options.words_file) train_file = numberized_file if options.mmap: train_file += '.mmap' @@ -139,16 +149,34 @@ def main(options): '--ngramize', '1', '--ngram_size', str(options.ngram_size), '--vocab_size', str(options.vocab_size), - '--write_words_file', os.path.join( - options.working_dir, options.words_file), + '--write_words_file', vocab_file, '--train_file', os.path.join(options.working_dir, numberized_file) ] sys.stderr.write('extracting n-grams\n') sys.stderr.write('executing: ' + ', '.join(extraction_cmd) + '\n') - ret = subprocess.call(extraction_cmd) - if ret: - raise Exception("preparing neural LM failed") + subprocess.check_call(extraction_cmd) + + # if dropout enabled, need to check which is the <null> vocab id + null_id = None + if options.dropout or options.input_dropout: + with open(vocab_file) as vfh: + for i,line in enumerate(vfh): + if line[:-1].decode("utf8") == "<null>": + null_id = i + break + if null_id == None: + sys.stderr.write("WARN: could not identify null token, cannot enable dropout\n") + else: + if not options.extra_settings: + options.extra_settings = "" + if options.dropout or options.input_dropout: + options.extra_settings += " --null_index %d " % null_id + if options.dropout: + options.extra_settings += " --dropout %s " % options.dropout + if options.input_dropout: + options.extra_settings += " --input_dropout %s " % options.input_dropout + if options.mmap: try: @@ -176,8 +204,7 @@ def main(options): '--ngramize', '1', '--ngram_size', str(options.ngram_size), '--vocab_size', str(options.vocab_size), - '--words_file', os.path.join( - options.working_dir, options.words_file), + '--words_file', vocab_file, '--train_file', os.path.join( options.working_dir, os.path.basename(options.validation_corpus) + '.numberized') @@ -194,8 +221,8 @@ def main(options): else: options.validation_file = None - options.input_words_file = os.path.join(options.working_dir, options.words_file) - options.output_words_file = os.path.join(options.working_dir, options.words_file) + options.input_words_file = vocab_file + options.output_words_file = vocab_file options.input_vocab_size = options.vocab_size options.output_vocab_size = options.vocab_size diff --git a/util/exception.cc b/util/exception.cc index 588f5eae5..5ba06f065 100644 --- a/util/exception.cc +++ b/util/exception.cc @@ -7,47 +7,42 @@ #include <cerrno> #include <cstring> +#if defined(_WIN32) || defined(_WIN64) +#include <windows.h> +#include <io.h> +#endif + namespace util { Exception::Exception() throw() {} Exception::~Exception() throw() {} -Exception::Exception(const Exception &from) : std::exception() { - stream_ << from.stream_.str(); -} - -Exception &Exception::operator=(const Exception &from) { - stream_ << from.stream_.str(); - return *this; -} - -const char *Exception::what() const throw() { - text_ = stream_.str(); - return text_.c_str(); -} - void Exception::SetLocation(const char *file, unsigned int line, const char *func, const char *child_name, const char *condition) { /* The child class might have set some text, but we want this to come first. * Another option would be passing this information to the constructor, but * then child classes would have to accept constructor arguments and pass * them down. */ - text_ = stream_.str(); - stream_.str(""); - stream_ << file << ':' << line; - if (func) stream_ << " in " << func << " threw "; + std::string old_text; + std::swap(old_text, what_); + StringStream stream; + stream << what_; + stream << file << ':' << line; + if (func) stream << " in " << func << " threw "; if (child_name) { - stream_ << child_name; + stream << child_name; } else { #ifdef __GXX_RTTI - stream_ << typeid(this).name(); + stream << typeid(this).name(); #else - stream_ << "an exception"; + stream << "an exception"; #endif } - if (condition) stream_ << " because `" << condition; - stream_ << "'.\n"; - stream_ << text_; + if (condition) { + stream << " because `" << condition << '\''; + } + stream << ".\n"; + stream << old_text; } namespace { @@ -95,4 +90,17 @@ ErrnoException::~ErrnoException() throw() {} OverflowException::OverflowException() throw() {} OverflowException::~OverflowException() throw() {} +#if defined(_WIN32) || defined(_WIN64) +WindowsException::WindowsException() throw() { + unsigned int last_error = GetLastError(); + char error_msg[256] = ""; + if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, last_error, LANG_NEUTRAL, error_msg, sizeof(error_msg), NULL)) { + *this << "Windows error " << GetLastError() << " while formatting Windows error " << last_error << ". "; + } else { + *this << "Windows error " << last_error << ": " << error_msg; + } +} +WindowsException::~WindowsException() throw() {} +#endif + } // namespace util diff --git a/util/exception.hh b/util/exception.hh index d67a6f9fb..00207b242 100644 --- a/util/exception.hh +++ b/util/exception.hh @@ -1,12 +1,16 @@ #ifndef UTIL_EXCEPTION_H #define UTIL_EXCEPTION_H +#include "util/string_stream.hh" + #include <exception> #include <limits> -#include <sstream> #include <string> #include <stdint.h> +// TODO(hieu) delete this +#include <sstream> + namespace util { template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data); @@ -16,11 +20,7 @@ class Exception : public std::exception { Exception() throw(); virtual ~Exception() throw(); - Exception(const Exception &from); - Exception &operator=(const Exception &from); - - // Not threadsafe, but probably doesn't matter. FWIW, Boost's exception guidance implies that what() isn't threadsafe. - const char *what() const throw(); + const char *what() const throw() { return what_.c_str(); } // For use by the UTIL_THROW macros. void SetLocation( @@ -38,8 +38,7 @@ class Exception : public std::exception { typedef T Identity; }; - std::stringstream stream_; - mutable std::string text_; + std::string what_; }; /* This implements the normal operator<< for Exception and all its children. @@ -47,7 +46,12 @@ class Exception : public std::exception { * boost::enable_if. */ template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data) { - e.stream_ << data; + // TODO(hieu): change this to + // StringStream(e.what_) << data; + + std::stringstream moses_hack; + moses_hack << data; + e.what_ += moses_hack.str(); return e; } @@ -149,6 +153,15 @@ inline std::size_t CheckOverflow(uint64_t value) { return CheckOverflowInternal<sizeof(std::size_t)>(value); } +#if defined(_WIN32) || defined(_WIN64) +/* Thrown for Windows specific operations. */ +class WindowsException : public Exception { + public: + WindowsException() throw(); + ~WindowsException() throw(); +}; +#endif + } // namespace util #endif // UTIL_EXCEPTION_H diff --git a/util/fake_ofstream.hh b/util/fake_ofstream.hh deleted file mode 100644 index d35bf0d83..000000000 --- a/util/fake_ofstream.hh +++ /dev/null @@ -1,137 +0,0 @@ -/* Like std::ofstream but without being incredibly slow. Backed by a raw fd. - * Supports most of the built-in types except for void* and long double. - */ -#ifndef UTIL_FAKE_OFSTREAM_H -#define UTIL_FAKE_OFSTREAM_H - -#include "util/file.hh" -#include "util/float_to_string.hh" -#include "util/integer_to_string.hh" -#include "util/scoped.hh" -#include "util/string_piece.hh" - -#include <cassert> -#include <cstring> - -#include <stdint.h> - -namespace util { -class FakeOFStream { - public: - // Maximum over all ToString operations. - // static const std::size_t kMinBuf = 20; - // This was causing compile failures in debug, so now 20 is written directly. - // - // Does not take ownership of out. - // Allows default constructor, but must call SetFD. - explicit FakeOFStream(int out = -1, std::size_t buffer_size = 1048576) - : buf_(util::MallocOrThrow(std::max(buffer_size, (size_t)20))), - current_(static_cast<char*>(buf_.get())), - end_(current_ + std::max(buffer_size, (size_t)20)), - fd_(out) {} - - ~FakeOFStream() { - // Could have called Finish already - flush(); - } - - void SetFD(int to) { - flush(); - fd_ = to; - } - - FakeOFStream &write(const void *data, std::size_t length) { - if (UTIL_LIKELY(current_ + length <= end_)) { - std::memcpy(current_, data, length); - current_ += length; - return *this; - } - flush(); - if (current_ + length <= end_) { - std::memcpy(current_, data, length); - current_ += length; - } else { - util::WriteOrThrow(fd_, data, length); - } - return *this; - } - - // This also covers std::string and char* - FakeOFStream &operator<<(StringPiece str) { - return write(str.data(), str.size()); - } - - // For anything with ToStringBuf<T>::kBytes, define operator<< using ToString. - // This includes uint64_t, int64_t, uint32_t, int32_t, uint16_t, int16_t, - // float, double - private: - template <int Arg> struct EnableIfKludge { - typedef FakeOFStream type; - }; - public: - template <class T> typename EnableIfKludge<ToStringBuf<T>::kBytes>::type &operator<<(const T value) { - EnsureRemaining(ToStringBuf<T>::kBytes); - current_ = ToString(value, current_); - assert(current_ <= end_); - return *this; - } - - FakeOFStream &operator<<(char c) { - EnsureRemaining(1); - *current_++ = c; - return *this; - } - - FakeOFStream &operator<<(unsigned char c) { - EnsureRemaining(1); - *current_++ = static_cast<char>(c); - return *this; - } - - /* clang on OS X appears to consider std::size_t aka unsigned long distinct - * from uint64_t. So this function makes clang work. gcc considers - * uint64_t and std::size_t the same (on 64-bit) so this isn't necessary. - * But it does no harm since gcc sees it as a specialization of the - * EnableIfKludge template. - * Also, delegating to *this << static_cast<uint64_t>(value) would loop - * indefinitely on gcc. - */ - FakeOFStream &operator<<(std::size_t value) { - EnsureRemaining(ToStringBuf<uint64_t>::kBytes); - current_ = ToString(static_cast<uint64_t>(value), current_); - return *this; - } - - // Note this does not sync. - void flush() { - if (current_ != buf_.get()) { - util::WriteOrThrow(fd_, buf_.get(), current_ - (char*)buf_.get()); - current_ = static_cast<char*>(buf_.get()); - } - } - - // Not necessary, but does assure the data is cleared. - void Finish() { - flush(); - buf_.reset(); - current_ = NULL; - util::FSyncOrThrow(fd_); - } - - private: - void EnsureRemaining(std::size_t amount) { - if (UTIL_UNLIKELY(current_ + amount > end_)) { - flush(); - assert(current_ + amount <= end_); - } - } - - util::scoped_malloc buf_; - char *current_, *end_; - - int fd_; -}; - -} // namespace - -#endif diff --git a/util/fake_ostream.hh b/util/fake_ostream.hh new file mode 100644 index 000000000..2f76053cc --- /dev/null +++ b/util/fake_ostream.hh @@ -0,0 +1,111 @@ +#ifndef UTIL_FAKE_OSTREAM_H +#define UTIL_FAKE_OSTREAM_H + +#include "util/float_to_string.hh" +#include "util/integer_to_string.hh" +#include "util/string_piece.hh" + +#include <cassert> +#include <limits> + +#include <stdint.h> + +namespace util { + +/* Like std::ostream but without being incredibly slow. + * Supports most of the built-in types except for long double. + * + * The FakeOStream class is intended to be inherited from. The inherting class + * should provide: + * public: + * Derived &flush(); + * Derived &write(const void *data, std::size_t length); + * + * private: or protected: + * friend class FakeOStream; + * char *Ensure(std::size_t amount); + * void AdvanceTo(char *to); + * + * The Ensure function makes enough space for an in-place write and returns + * where to write. The AdvanceTo function happens after the write, saying how + * much was actually written. + * + * Precondition: + * amount <= kToStringMaxBytes for in-place writes. + */ +template <class Derived> class FakeOStream { + public: + FakeOStream() {} + + // This also covers std::string and char* + Derived &operator<<(StringPiece str) { + return C().write(str.data(), str.size()); + } + + // Handle integers by size and signedness. + private: + template <class Arg> struct EnableIfKludge { + typedef Derived type; + }; + template <class From, unsigned Length = sizeof(From), bool Signed = std::numeric_limits<From>::is_signed, bool IsInteger = std::numeric_limits<From>::is_integer> struct Coerce {}; + + template <class From> struct Coerce<From, 2, false, true> { typedef uint16_t To; }; + template <class From> struct Coerce<From, 4, false, true> { typedef uint32_t To; }; + template <class From> struct Coerce<From, 8, false, true> { typedef uint64_t To; }; + + template <class From> struct Coerce<From, 2, true, true> { typedef int16_t To; }; + template <class From> struct Coerce<From, 4, true, true> { typedef int32_t To; }; + template <class From> struct Coerce<From, 8, true, true> { typedef int64_t To; }; + public: + template <class From> typename EnableIfKludge<typename Coerce<From>::To>::type &operator<<(const From value) { + return CallToString(static_cast<typename Coerce<From>::To>(value)); + } + + // Character types that get copied as bytes instead of displayed as integers. + Derived &operator<<(char val) { return put(val); } + Derived &operator<<(signed char val) { return put(static_cast<char>(val)); } + Derived &operator<<(unsigned char val) { return put(static_cast<char>(val)); } + + Derived &operator<<(bool val) { return put(val + '0'); } + // enums will fall back to int but are not caught by the template. + Derived &operator<<(int val) { return CallToString(static_cast<typename Coerce<int>::To>(val)); } + + Derived &operator<<(float val) { return CallToString(val); } + Derived &operator<<(double val) { return CallToString(val); } + + // This is here to catch all the other pointer types. + Derived &operator<<(const void *value) { return CallToString(value); } + // This is here because the above line also catches const char*. + Derived &operator<<(const char *value) { return *this << StringPiece(value); } + Derived &operator<<(char *value) { return *this << StringPiece(value); } + + Derived &put(char val) { + char *c = C().Ensure(1); + *c = val; + C().AdvanceTo(++c); + return C(); + } + + char widen(char val) const { return val; } + + private: + // References to derived class for convenience. + Derived &C() { + return *static_cast<Derived*>(this); + } + + const Derived &C() const { + return *static_cast<const Derived*>(this); + } + + // This is separate to prevent an infinite loop if the compiler considers + // types the same (i.e. gcc std::size_t and uint64_t or uint32_t). + template <class T> Derived &CallToString(const T value) { + C().AdvanceTo(ToString(value, C().Ensure(ToStringBuf<T>::kBytes))); + return C(); + } +}; + +} // namespace + +#endif // UTIL_FAKE_OSTREAM_H diff --git a/util/file.cc b/util/file.cc index be272f9bc..e8976bc10 100644 --- a/util/file.cc +++ b/util/file.cc @@ -147,17 +147,33 @@ std::size_t GuardLarge(std::size_t size) { } } +#if defined(_WIN32) || defined(_WIN64) +namespace { +const std::size_t kMaxDWORD = static_cast<std::size_t>(4294967295UL); +} // namespace +#endif + std::size_t PartialRead(int fd, void *to, std::size_t amount) { #if defined(_WIN32) || defined(_WIN64) - int ret = _read(fd, to, GuardLarge(amount)); + DWORD ret; + HANDLE file_handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd)); + DWORD larger_size = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, amount)); + DWORD smaller_size = 28672; // Received reports that 31346 worked but higher values did not. This rounds down to the nearest multiple of 4096, the page size. + if (!ReadFile(file_handle, to, larger_size, &ret, NULL)) + { + DWORD last_error = GetLastError(); + if (last_error != ERROR_NOT_ENOUGH_MEMORY || !ReadFile(file_handle, to, smaller_size, &ret, NULL)) { + UTIL_THROW(WindowsException, "Windows error in ReadFile."); + } + } #else errno = 0; ssize_t ret; do { ret = read(fd, to, GuardLarge(amount)); } while (ret == -1 && errno == EINTR); -#endif UTIL_THROW_IF_ARG(ret < 0, FDException, (fd), "while reading " << amount << " bytes"); +#endif return static_cast<std::size_t>(ret); } @@ -212,12 +228,6 @@ void WriteOrThrow(FILE *to, const void *data, std::size_t size) { UTIL_THROW_IF(1 != std::fwrite(data, size, 1, to), ErrnoException, "Short write; requested size " << size); } -#if defined(_WIN32) || defined(_WIN64) -namespace { -const std::size_t kMaxDWORD = static_cast<std::size_t>(4294967295UL); -} // namespace -#endif - void ErsatzPRead(int fd, void *to_void, std::size_t size, uint64_t off) { uint8_t *to = static_cast<uint8_t*>(to_void); while (size) { @@ -230,7 +240,7 @@ void ErsatzPRead(int fd, void *to_void, std::size_t size, uint64_t off) { memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = static_cast<DWORD>(off); overlapped.OffsetHigh = static_cast<DWORD>(off >> 32); - UTIL_THROW_IF(!ReadFile((HANDLE)_get_osfhandle(fd), to, reading, &ret, &overlapped), Exception, "ReadFile failed for offset " << off); + UTIL_THROW_IF(!ReadFile((HANDLE)_get_osfhandle(fd), to, reading, &ret, &overlapped), WindowsException, "ReadFile failed for offset " << off); #else ssize_t ret; errno = 0; diff --git a/util/file_piece.cc b/util/file_piece.cc index 7017942e6..0a4d3a9da 100644 --- a/util/file_piece.cc +++ b/util/file_piece.cc @@ -56,7 +56,7 @@ FilePiece::FilePiece(std::istream &stream, const char *name, std::size_t min_buf InitializeNoRead("istream", min_buffer); fallback_to_read_ = true; - data_.reset(MallocOrThrow(default_map_size_), default_map_size_, scoped_memory::MALLOC_ALLOCATED); + HugeMalloc(default_map_size_, false, data_); position_ = data_.begin(); position_end_ = position_; @@ -282,7 +282,7 @@ void FilePiece::TransitionToRead() { assert(!fallback_to_read_); fallback_to_read_ = true; data_.reset(); - data_.reset(MallocOrThrow(default_map_size_), default_map_size_, scoped_memory::MALLOC_ALLOCATED); + HugeMalloc(default_map_size_, false, data_); position_ = data_.begin(); position_end_ = position_; @@ -313,8 +313,7 @@ void FilePiece::ReadShift() { // Buffer too small. std::size_t valid_length = position_end_ - position_; default_map_size_ *= 2; - data_.call_realloc(default_map_size_); - UTIL_THROW_IF(!data_.get(), ErrnoException, "realloc failed for " << default_map_size_); + HugeRealloc(default_map_size_, false, data_); position_ = data_.begin(); position_end_ = position_ + valid_length; } else { diff --git a/util/file_piece_test.cc b/util/file_piece_test.cc index 11e2ab3aa..d03cd312d 100644 --- a/util/file_piece_test.cc +++ b/util/file_piece_test.cc @@ -1,7 +1,7 @@ // Tests might fail if you have creative characters in your path. Sue me. #include "util/file_piece.hh" -#include "util/fake_ofstream.hh" +#include "util/file_stream.hh" #include "util/file.hh" #include "util/scoped.hh" @@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE(Numbers) { scoped_fd file(MakeTemp(FileLocation())); const float floating = 3.2; { - util::FakeOFStream writing(file.get()); + util::FileStream writing(file.get()); writing << "94389483984398493890287 " << floating << " 5"; } SeekOrThrow(file.get(), 0); diff --git a/util/file_stream.hh b/util/file_stream.hh new file mode 100644 index 000000000..ae9ad5aa7 --- /dev/null +++ b/util/file_stream.hh @@ -0,0 +1,89 @@ +/* Like std::ofstream but without being incredibly slow. Backed by a raw fd. + * Supports most of the built-in types except for long double. + */ +#ifndef UTIL_FILE_STREAM_H +#define UTIL_FILE_STREAM_H + +#include "util/fake_ostream.hh" +#include "util/file.hh" +#include "util/scoped.hh" + +#include <cassert> +#include <cstring> + +#include <stdint.h> + +namespace util { + +class FileStream : public FakeOStream<FileStream> { + public: + FileStream(int out = -1, std::size_t buffer_size = 8192) + : buf_(util::MallocOrThrow(std::max<std::size_t>(buffer_size, kToStringMaxBytes))), + current_(static_cast<char*>(buf_.get())), + end_(current_ + std::max<std::size_t>(buffer_size, kToStringMaxBytes)), + fd_(out) {} + + ~FileStream() { + flush(); + } + + void SetFD(int to) { + flush(); + fd_ = to; + } + + FileStream &flush() { + if (current_ != buf_.get()) { + util::WriteOrThrow(fd_, buf_.get(), current_ - (char*)buf_.get()); + current_ = static_cast<char*>(buf_.get()); + } + return *this; + } + + // For writes of arbitrary size. + FileStream &write(const void *data, std::size_t length) { + if (UTIL_LIKELY(current_ + length <= end_)) { + std::memcpy(current_, data, length); + current_ += length; + return *this; + } + flush(); + if (current_ + length <= end_) { + std::memcpy(current_, data, length); + current_ += length; + } else { + util::WriteOrThrow(fd_, data, length); + } + return *this; + } + + FileStream &seekp(uint64_t to) { + util::SeekOrThrow(fd_, to); + return *this; + } + + protected: + friend class FakeOStream<FileStream>; + // For writes directly to buffer guaranteed to have amount < buffer size. + char *Ensure(std::size_t amount) { + if (UTIL_UNLIKELY(current_ + amount > end_)) { + flush(); + assert(current_ + amount <= end_); + } + return current_; + } + + void AdvanceTo(char *to) { + current_ = to; + assert(current_ <= end_); + } + + private: + util::scoped_malloc buf_; + char *current_, *end_; + int fd_; +}; + +} // namespace + +#endif diff --git a/util/integer_to_string.cc b/util/integer_to_string.cc index 6b8766119..19fd794e6 100644 --- a/util/integer_to_string.cc +++ b/util/integer_to_string.cc @@ -1,3 +1,4 @@ +#include <iostream> /* Fast integer to string conversion. Source: https://github.com/miloyip/itoa-benchmark Local modifications: @@ -637,4 +638,30 @@ char *ToString(uint16_t value, char *to) { return ToString((uint32_t)value, to); } +// void * to string. This hasn't been optimized at all really. +namespace { +const char kHexDigits[] = "0123456789abcdef"; +} // namespace + +char *ToString(const void *v, char *to) { + *to++ = '0'; + *to++ = 'x'; + + // Fun fact: gcc/clang boost::lexical_cast on Linux do just "0" while clang on OS X does "0x0" + // I happen to prefer 0x0. + if (!v) { + *to++ = '0'; + return to; + } + + uintptr_t value = reinterpret_cast<uintptr_t>(v); + uint8_t shift = sizeof(void*) * 8 - 4; + for (; !(value >> shift); shift -= 4) {} + for (; ; shift -= 4) { + *to++ = kHexDigits[(value >> shift) & 0xf]; + if (!shift) break; + } + return to; +} + } // namespace util diff --git a/util/integer_to_string.hh b/util/integer_to_string.hh index 0d975b14e..9ac25bd78 100644 --- a/util/integer_to_string.hh +++ b/util/integer_to_string.hh @@ -18,6 +18,8 @@ char *ToString(int64_t value, char *to); char *ToString(uint16_t value, char *to); char *ToString(int16_t value, char *to); +char *ToString(const void *value, char *to); + inline char *ToString(bool value, char *to) { *to++ = '0' + value; return to; @@ -51,6 +53,14 @@ template <> struct ToStringBuf<int64_t> { enum { kBytes = 20 }; }; +template <> struct ToStringBuf<const void*> { + // Either 18 on 64-bit or 10 on 32-bit. + enum { kBytes = sizeof(const void*) * 2 + 2 }; +}; + +// Maximum over this and float. +enum { kToStringMaxBytes = 20 }; + } // namespace util #endif // UTIL_INTEGER_TO_STRING_H diff --git a/util/integer_to_string_test.cc b/util/integer_to_string_test.cc index ded1ecec7..136c88f62 100644 --- a/util/integer_to_string_test.cc +++ b/util/integer_to_string_test.cc @@ -15,15 +15,20 @@ template <class T> void TestValue(const T value) { char buf[ToStringBuf<T>::kBytes]; StringPiece result(buf, ToString(value, buf) - buf); BOOST_REQUIRE_GE(static_cast<std::size_t>(ToStringBuf<T>::kBytes), result.size()); - BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), result); + if (value) { + BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), result); + } else { + // Platforms can do void * as 0x0 or 0. + BOOST_CHECK(result == "0x0" || result == "0"); + } } template <class T> void TestCorners() { TestValue(std::numeric_limits<T>::min()); TestValue(std::numeric_limits<T>::max()); - TestValue(static_cast<T>(0)); - TestValue(static_cast<T>(-1)); - TestValue(static_cast<T>(1)); + TestValue((T)0); + TestValue((T)-1); + TestValue((T)1); } BOOST_AUTO_TEST_CASE(Corners) { @@ -33,6 +38,7 @@ BOOST_AUTO_TEST_CASE(Corners) { TestCorners<int16_t>(); TestCorners<int32_t>(); TestCorners<int64_t>(); + TestCorners<const void*>(); } template <class T> void TestAll() { @@ -62,4 +68,14 @@ BOOST_AUTO_TEST_CASE(Tens) { Test10s<int32_t>(); } +BOOST_AUTO_TEST_CASE(Pointers) { + for (uintptr_t i = 1; i < std::numeric_limits<uintptr_t>::max() / 10; i *= 10) { + TestValue((const void*)i); + } + for (uintptr_t i = 0; i < 256; ++i) { + TestValue((const void*)i); + TestValue((const void*)(i + 0xf00)); + } +} + }} // namespaces diff --git a/util/mmap.cc b/util/mmap.cc index 7dcb57ba3..d89f3f600 100644 --- a/util/mmap.cc +++ b/util/mmap.cc @@ -27,7 +27,7 @@ namespace util { -long SizePage() { +std::size_t SizePage() { #if defined(_WIN32) || defined(_WIN64) SYSTEM_INFO si; GetSystemInfo(&si); @@ -37,22 +37,6 @@ long SizePage() { #endif } -void SyncOrThrow(void *start, size_t length) { -#if defined(_WIN32) || defined(_WIN64) - UTIL_THROW_IF(!::FlushViewOfFile(start, length), ErrnoException, "Failed to sync mmap"); -#else - UTIL_THROW_IF(length && msync(start, length, MS_SYNC), ErrnoException, "Failed to sync mmap"); -#endif -} - -void UnmapOrThrow(void *start, size_t length) { -#if defined(_WIN32) || defined(_WIN64) - UTIL_THROW_IF(!::UnmapViewOfFile(start), ErrnoException, "Failed to unmap a file"); -#else - UTIL_THROW_IF(munmap(start, length), ErrnoException, "munmap failed"); -#endif -} - scoped_mmap::~scoped_mmap() { if (data_ != (void*)-1) { try { @@ -66,14 +50,24 @@ scoped_mmap::~scoped_mmap() { } } +namespace { +template <class T> T RoundUpPow2(T value, T mult) { + return ((value - 1) & ~(mult - 1)) + mult; +} +} // namespace + +scoped_memory::scoped_memory(std::size_t size, bool zeroed) : data_(NULL), size_(0), source_(NONE_ALLOCATED) { + HugeMalloc(size, zeroed, *this); +} + void scoped_memory::reset(void *data, std::size_t size, Alloc source) { switch(source_) { + case MMAP_ROUND_UP_ALLOCATED: + scoped_mmap(data_, RoundUpPow2(size_, (std::size_t)SizePage())); + break; case MMAP_ALLOCATED: scoped_mmap(data_, size_); break; - case ARRAY_ALLOCATED: - delete [] reinterpret_cast<char*>(data_); - break; case MALLOC_ALLOCATED: free(data_); break; @@ -85,7 +79,7 @@ void scoped_memory::reset(void *data, std::size_t size, Alloc source) { source_ = source; } -void scoped_memory::call_realloc(std::size_t size) { +/*void scoped_memory::call_realloc(std::size_t size) { assert(source_ == MALLOC_ALLOCATED || source_ == NONE_ALLOCATED); void *new_data = realloc(data_, size); if (!new_data) { @@ -95,7 +89,17 @@ void scoped_memory::call_realloc(std::size_t size) { size_ = size; source_ = MALLOC_ALLOCATED; } -} +}*/ + +const int kFileFlags = +#if defined(_WIN32) || defined(_WIN64) + 0 // MapOrThrow ignores flags on windows +#elif defined(MAP_FILE) + MAP_FILE | MAP_SHARED +#else + MAP_SHARED +#endif + ; void *MapOrThrow(std::size_t size, bool for_write, int flags, bool prefault, int fd, uint64_t offset) { #ifdef MAP_POPULATE // Linux specific @@ -126,15 +130,170 @@ void *MapOrThrow(std::size_t size, bool for_write, int flags, bool prefault, int return ret; } -const int kFileFlags = +void SyncOrThrow(void *start, size_t length) { #if defined(_WIN32) || defined(_WIN64) - 0 // MapOrThrow ignores flags on windows -#elif defined(MAP_FILE) - MAP_FILE | MAP_SHARED + UTIL_THROW_IF(!::FlushViewOfFile(start, length), ErrnoException, "Failed to sync mmap"); #else - MAP_SHARED + UTIL_THROW_IF(length && msync(start, length, MS_SYNC), ErrnoException, "Failed to sync mmap"); #endif - ; +} + +void UnmapOrThrow(void *start, size_t length) { +#if defined(_WIN32) || defined(_WIN64) + UTIL_THROW_IF(!::UnmapViewOfFile(start), ErrnoException, "Failed to unmap a file"); +#else + UTIL_THROW_IF(munmap(start, length), ErrnoException, "munmap failed"); +#endif +} + +// Linux huge pages. +#ifdef __linux__ + +namespace { + +bool AnonymousMap(std::size_t size, int flags, bool populate, util::scoped_memory &to) { + if (populate) flags |= MAP_POPULATE; + void *ret = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | flags, -1, 0); + if (ret == MAP_FAILED) return false; + to.reset(ret, size, scoped_memory::MMAP_ALLOCATED); + return true; +} + +bool TryHuge(std::size_t size, uint8_t alignment_bits, bool populate, util::scoped_memory &to) { + // Don't bother with these cases. + if (size < (1ULL << alignment_bits) || (1ULL << alignment_bits) < SizePage()) + return false; + + // First try: Linux >= 3.8 with manually configured hugetlb pages available. +#ifdef MAP_HUGE_SHIFT + if (AnonymousMap(size, MAP_HUGETLB | (alignment_bits << MAP_HUGE_SHIFT), populate, to)) + return true; +#endif + + // Second try: manually configured hugetlb pages exist, but kernel too old to + // pick size or not available. This might pick the wrong size huge pages, + // but the sysadmin must have made them available in the first place. + if (AnonymousMap(size, MAP_HUGETLB, populate, to)) + return true; + + // Third try: align to a multiple of the huge page size by overallocating. + // I feel bad about doing this, but it's also how posix_memalign is + // implemented. And the memory is virtual. + + // Round up requested size to multiple of page size. This will allow the pages after to be munmapped. + std::size_t size_up = RoundUpPow2(size, SizePage()); + + std::size_t ask = size_up + (1 << alignment_bits) - SizePage(); + // Don't populate because this is asking for more than we will use. + scoped_mmap larger(mmap(NULL, ask, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), ask); + if (larger.get() == MAP_FAILED) return false; + + // Throw out pages before the alignment point. + uintptr_t base = reinterpret_cast<uintptr_t>(larger.get()); + // Round up to next multiple of alignment. + uintptr_t rounded_up = RoundUpPow2(base, static_cast<uintptr_t>(1) << alignment_bits); + if (base != rounded_up) { + // If this throws an exception (which it shouldn't) then we want to unmap the whole thing by keeping it in larger. + UnmapOrThrow(larger.get(), rounded_up - base); + larger.steal(); + larger.reset(reinterpret_cast<void*>(rounded_up), ask - (rounded_up - base)); + } + + // Throw out pages after the requested size. + assert(larger.size() >= size_up); + if (larger.size() > size_up) { + // This is where we assume size_up is a multiple of page size. + UnmapOrThrow(static_cast<uint8_t*>(larger.get()) + size_up, larger.size() - size_up); + larger.reset(larger.steal(), size_up); + } +#ifdef MADV_HUGEPAGE + madvise(larger.get(), size_up, MADV_HUGEPAGE); +#endif + to.reset(larger.steal(), size, scoped_memory::MMAP_ROUND_UP_ALLOCATED); + return true; +} + +} // namespace + +#endif + +void HugeMalloc(std::size_t size, bool zeroed, scoped_memory &to) { + to.reset(); +#ifdef __linux__ + // TODO: architectures/page sizes other than 2^21 and 2^30. + // Attempt 1 GB pages. + // If the user asked for zeroed memory, assume they want it populated. + if (size >= (1ULL << 30) && TryHuge(size, 30, zeroed, to)) + return; + // Attempt 2 MB pages. + if (size >= (1ULL << 21) && TryHuge(size, 21, zeroed, to)) + return; +#endif // __linux__ + // Non-linux will always do this, as will small allocations on Linux. + to.reset(zeroed ? calloc(1, size) : malloc(size), size, scoped_memory::MALLOC_ALLOCATED); + UTIL_THROW_IF(!to.get(), ErrnoException, "Failed to allocate " << size << " bytes"); +} + +#ifdef __linux__ +const std::size_t kTransitionHuge = std::max<std::size_t>(1ULL << 21, SizePage()); +#endif // __linux__ + +void HugeRealloc(std::size_t to, bool zero_new, scoped_memory &mem) { + if (!to) { + mem.reset(); + return; + } + std::size_t from_size = mem.size(); + switch (mem.source()) { + case scoped_memory::NONE_ALLOCATED: + HugeMalloc(to, zero_new, mem); + return; +#ifdef __linux__ + case scoped_memory::MMAP_ROUND_UP_ALLOCATED: + // for mremap's benefit. + from_size = RoundUpPow2(from_size, SizePage()); + case scoped_memory::MMAP_ALLOCATED: + // Downsizing below barrier? + if (to <= SizePage()) { + scoped_malloc replacement(malloc(to)); + memcpy(replacement.get(), mem.get(), std::min(to, mem.size())); + if (zero_new && to > mem.size()) + memset(static_cast<uint8_t*>(replacement.get()) + mem.size(), 0, to - mem.size()); + mem.reset(replacement.release(), to, scoped_memory::MALLOC_ALLOCATED); + } else { + void *new_addr = mremap(mem.get(), from_size, to, MREMAP_MAYMOVE); + UTIL_THROW_IF(!new_addr, ErrnoException, "Failed to mremap from " << from_size << " to " << to); + mem.steal(); + mem.reset(new_addr, to, scoped_memory::MMAP_ALLOCATED); + } + return; +#endif // __linux__ + case scoped_memory::MALLOC_ALLOCATED: +#ifdef __linux__ + // Transition larger allocations to huge pages, but don't keep trying if we're still malloc allocated. + if (to >= kTransitionHuge && mem.size() < kTransitionHuge) { + scoped_memory replacement; + HugeMalloc(to, zero_new, replacement); + memcpy(replacement.get(), mem.get(), mem.size()); + // This can't throw. + mem.reset(replacement.get(), replacement.size(), replacement.source()); + replacement.steal(); + return; + } +#endif // __linux__ + { + void *new_addr = std::realloc(mem.get(), to); + UTIL_THROW_IF(!new_addr, ErrnoException, "realloc to " << to << " bytes failed."); + if (zero_new && to > mem.size()) + memset(static_cast<uint8_t*>(new_addr) + mem.size(), 0, to - mem.size()); + mem.steal(); + mem.reset(new_addr, to, scoped_memory::MALLOC_ALLOCATED); + } + return; + default: + UTIL_THROW(Exception, "HugeRealloc called with type " << mem.source()); + } +} void MapRead(LoadMethod method, int fd, uint64_t offset, std::size_t size, scoped_memory &out) { switch (method) { @@ -151,33 +310,17 @@ void MapRead(LoadMethod method, int fd, uint64_t offset, std::size_t size, scope case POPULATE_OR_READ: #endif case READ: - out.reset(MallocOrThrow(size), size, scoped_memory::MALLOC_ALLOCATED); + HugeMalloc(size, false, out); SeekOrThrow(fd, offset); ReadOrThrow(fd, out.get(), size); break; case PARALLEL_READ: - out.reset(MallocOrThrow(size), size, scoped_memory::MALLOC_ALLOCATED); + HugeMalloc(size, false, out); ParallelRead(fd, out.get(), size, offset); break; } } -// Allocates zeroed memory in to. -void MapAnonymous(std::size_t size, util::scoped_memory &to) { - to.reset(); -#if defined(_WIN32) || defined(_WIN64) - to.reset(calloc(1, size), size, scoped_memory::MALLOC_ALLOCATED); -#else - to.reset(MapOrThrow(size, true, -# if defined(MAP_ANONYMOUS) - MAP_ANONYMOUS | MAP_PRIVATE // Linux -# else - MAP_ANON | MAP_PRIVATE // BSD -# endif - , false, -1, 0), size, scoped_memory::MMAP_ALLOCATED); -#endif -} - void *MapZeroedWrite(int fd, std::size_t size) { ResizeOrThrow(fd, 0); ResizeOrThrow(fd, size); diff --git a/util/mmap.hh b/util/mmap.hh index 9ac604975..b474dc75b 100644 --- a/util/mmap.hh +++ b/util/mmap.hh @@ -12,7 +12,7 @@ namespace util { class scoped_fd; -long SizePage(); +std::size_t SizePage(); // (void*)-1 is MAP_FAILED; this is done to avoid including the mmap header here. class scoped_mmap { @@ -37,6 +37,13 @@ class scoped_mmap { reset((void*)-1, 0); } + void *steal() { + void *ret = data_; + data_ = (void*)-1; + size_ = 0; + return ret; + } + private: void *data_; std::size_t size_; @@ -51,13 +58,21 @@ class scoped_mmap { */ class scoped_memory { public: - typedef enum {MMAP_ALLOCATED, ARRAY_ALLOCATED, MALLOC_ALLOCATED, NONE_ALLOCATED} Alloc; + typedef enum { + MMAP_ROUND_UP_ALLOCATED, // The size was rounded up to a multiple of page size. Do the same before munmap. + MMAP_ALLOCATED, // munmap + MALLOC_ALLOCATED, // free + NONE_ALLOCATED // nothing here! + } Alloc; scoped_memory(void *data, std::size_t size, Alloc source) : data_(data), size_(size), source_(source) {} scoped_memory() : data_(NULL), size_(0), source_(NONE_ALLOCATED) {} + // Calls HugeMalloc + scoped_memory(std::size_t to, bool zero_new); + ~scoped_memory() { reset(); } void *get() const { return data_; } @@ -71,9 +86,13 @@ class scoped_memory { void reset(void *data, std::size_t size, Alloc from); - // realloc allows the current data to escape hence the need for this call - // If realloc fails, destroys the original too and get() returns NULL. - void call_realloc(std::size_t to); + void *steal() { + void *ret = data_; + data_ = NULL; + size_ = 0; + source_ = NONE_ALLOCATED; + return ret; + } private: void *data_; @@ -85,6 +104,30 @@ class scoped_memory { scoped_memory &operator=(const scoped_memory &); }; +extern const int kFileFlags; + +// Cross-platform, error-checking wrapper for mmap(). +void *MapOrThrow(std::size_t size, bool for_write, int flags, bool prefault, int fd, uint64_t offset = 0); + +// msync wrapper +void SyncOrThrow(void *start, size_t length); + +// Cross-platform, error-checking wrapper for munmap(). +void UnmapOrThrow(void *start, size_t length); + +// Allocate memory, promising that all/vast majority of it will be used. Tries +// hard to use huge pages on Linux. +// If you want zeroed memory, pass zeroed = true. +void HugeMalloc(std::size_t size, bool zeroed, scoped_memory &to); + +// Reallocates memory ala realloc but with option to zero the new memory. +// On Linux, the memory can come from anonymous mmap or malloc/calloc. +// On non-Linux, only malloc/calloc is supported. +// +// To summarize, any memory from HugeMalloc or HugeRealloc can be resized with +// this. +void HugeRealloc(std::size_t size, bool new_zeroed, scoped_memory &mem); + typedef enum { // mmap with no prepopulate LAZY, @@ -98,25 +141,12 @@ typedef enum { PARALLEL_READ, } LoadMethod; -extern const int kFileFlags; - -// Cross-platform, error-checking wrapper for mmap(). -void *MapOrThrow(std::size_t size, bool for_write, int flags, bool prefault, int fd, uint64_t offset = 0); - -// Cross-platform, error-checking wrapper for munmap(). -void UnmapOrThrow(void *start, size_t length); - void MapRead(LoadMethod method, int fd, uint64_t offset, std::size_t size, scoped_memory &out); -void MapAnonymous(std::size_t size, scoped_memory &to); - // Open file name with mmap of size bytes, all of which are initially zero. void *MapZeroedWrite(int fd, std::size_t size); void *MapZeroedWrite(const char *name, std::size_t size, scoped_fd &file); -// msync wrapper -void SyncOrThrow(void *start, size_t length); - // Forward rolling memory map with no overlap. class Rolling { public: diff --git a/util/pool.cc b/util/pool.cc index e0e4c61f6..246417c16 100644 --- a/util/pool.cc +++ b/util/pool.cc @@ -4,6 +4,8 @@ #include <cstdlib> +#include <algorithm> + namespace util { Pool::Pool() { diff --git a/util/probing_hash_table.hh b/util/probing_hash_table.hh index 53fbbe996..438de92fb 100644 --- a/util/probing_hash_table.hh +++ b/util/probing_hash_table.hh @@ -2,7 +2,7 @@ #define UTIL_PROBING_HASH_TABLE_H #include "util/exception.hh" -#include "util/scoped.hh" +#include "util/mmap.hh" #include <algorithm> #include <cstddef> @@ -296,6 +296,13 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry } } + ConstIterator RawBegin() const { + return begin_; + } + ConstIterator RawEnd() const { + return end_; + } + private: friend class AutoProbing<Entry, Hash, Equal>; @@ -336,9 +343,11 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry typedef EqualT Equal; AutoProbing(std::size_t initial_size = 5, const Key &invalid = Key(), const Hash &hash_func = Hash(), const Equal &equal_func = Equal()) : - allocated_(Backend::Size(initial_size, 1.5)), mem_(util::MallocOrThrow(allocated_)), backend_(mem_.get(), allocated_, invalid, hash_func, equal_func) { - threshold_ = initial_size * 1.2; - Clear(); + allocated_(Backend::Size(initial_size, 1.2)), mem_(allocated_, KeyIsRawZero(invalid)), backend_(mem_.get(), allocated_, invalid, hash_func, equal_func) { + threshold_ = std::min<std::size_t>(backend_.buckets_ - 1, backend_.buckets_ * 0.9); + if (!KeyIsRawZero(invalid)) { + Clear(); + } } // Assumes that the key is unique. Multiple insertions won't cause a failure, just inconsistent lookup. @@ -377,18 +386,32 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry backend_.Clear(); } + ConstIterator RawBegin() const { + return backend_.RawBegin(); + } + ConstIterator RawEnd() const { + return backend_.RawEnd(); + } + private: void DoubleIfNeeded() { - if (Size() < threshold_) + if (UTIL_LIKELY(Size() < threshold_)) return; - mem_.call_realloc(backend_.DoubleTo()); + HugeRealloc(backend_.DoubleTo(), KeyIsRawZero(backend_.invalid_), mem_); allocated_ = backend_.DoubleTo(); - backend_.Double(mem_.get()); - threshold_ *= 2; + backend_.Double(mem_.get(), !KeyIsRawZero(backend_.invalid_)); + threshold_ = std::min<std::size_t>(backend_.buckets_ - 1, backend_.buckets_ * 0.9); + } + + bool KeyIsRawZero(const Key &key) { + for (const uint8_t *i = reinterpret_cast<const uint8_t*>(&key); i < reinterpret_cast<const uint8_t*>(&key) + sizeof(Key); ++i) { + if (*i) return false; + } + return true; } std::size_t allocated_; - util::scoped_malloc mem_; + util::scoped_memory mem_; Backend backend_; std::size_t threshold_; }; diff --git a/util/probing_hash_table_benchmark_main.cc b/util/probing_hash_table_benchmark_main.cc index c5129480f..583d21f5e 100644 --- a/util/probing_hash_table_benchmark_main.cc +++ b/util/probing_hash_table_benchmark_main.cc @@ -1,6 +1,6 @@ #include "util/file.hh" #include "util/probing_hash_table.hh" -#include "util/scoped.hh" +#include "util/mmap.hh" #include "util/usage.hh" #include <iostream> @@ -46,11 +46,12 @@ struct PrefetchEntry { const Entry *pointer; }; -const std::size_t kPrefetchSize = 4; -template <class Table> class PrefetchQueue { +template <class TableT, unsigned PrefetchSize> class PrefetchQueue { public: + typedef TableT Table; + explicit PrefetchQueue(Table &table) : table_(table), cur_(0), twiddle_(false) { - for (PrefetchEntry *i = entries_; i != entries_ + kPrefetchSize; ++i) + for (PrefetchEntry *i = entries_; i != entries_ + PrefetchSize; ++i) i->pointer = NULL; } @@ -66,7 +67,7 @@ template <class Table> class PrefetchQueue { bool Drain() { if (Cur().pointer) { - for (PrefetchEntry *i = &Cur(); i < entries_ + kPrefetchSize; ++i) { + for (PrefetchEntry *i = &Cur(); i < entries_ + PrefetchSize; ++i) { twiddle_ ^= table_.FindFromIdeal(i->key, i->pointer); } } @@ -80,11 +81,11 @@ template <class Table> class PrefetchQueue { PrefetchEntry &Cur() { return entries_[cur_]; } void Next() { ++cur_; - cur_ = cur_ % kPrefetchSize; + cur_ = cur_ % PrefetchSize; } Table &table_; - PrefetchEntry entries_[kPrefetchSize]; + PrefetchEntry entries_[PrefetchSize]; std::size_t cur_; bool twiddle_; @@ -93,12 +94,23 @@ template <class Table> class PrefetchQueue { void operator=(const PrefetchQueue&); }; -/*template <class Table> class Immediate { +template <class TableT> class Immediate { public: + typedef TableT Table; + + explicit Immediate(Table &table) : table_(table), twiddle_(false) {} + + void Add(uint64_t key) { + typename Table::ConstIterator it; + twiddle_ ^= table_.Find(key, it); + } + + bool Drain() const { return twiddle_; } private: Table &table_; -};*/ + bool twiddle_; +}; std::size_t Size(uint64_t entries, float multiplier = 1.5) { typedef util::ProbingHashTable<Entry, util::IdentityHash, std::equal_to<Entry::Key>, Power2Mod> Table; @@ -106,39 +118,54 @@ std::size_t Size(uint64_t entries, float multiplier = 1.5) { return Power2Mod::RoundBuckets(Table::Size(entries, multiplier) / sizeof(Entry)) * sizeof(Entry); } -template <class Mod> bool Test(URandom &rn, uint64_t entries, const uint64_t *const queries_begin, const uint64_t *const queries_end, float multiplier = 1.5) { - typedef util::ProbingHashTable<Entry, util::IdentityHash, std::equal_to<Entry::Key>, Mod> Table; +template <class Queue> bool Test(URandom &rn, uint64_t entries, const uint64_t *const queries_begin, const uint64_t *const queries_end, bool ordinary_malloc, float multiplier = 1.5) { std::size_t size = Size(entries, multiplier); - scoped_malloc backing(util::CallocOrThrow(size)); - Table table(backing.get(), size); + scoped_memory backing; + if (ordinary_malloc) { + backing.reset(util::CallocOrThrow(size), size, scoped_memory::MALLOC_ALLOCATED); + } else { + util::HugeMalloc(size, true, backing); + } + typename Queue::Table table(backing.get(), size); - double start = UserTime(); + double start = CPUTime(); for (uint64_t i = 0; i < entries; ++i) { Entry entry; entry.key = rn.Get(); table.Insert(entry); } - double inserted = UserTime() - start; - double before_lookup = UserTime(); - PrefetchQueue<Table> queue(table); + double inserted = CPUTime() - start; + double before_lookup = CPUTime(); + Queue queue(table); for (const uint64_t *i = queries_begin; i != queries_end; ++i) { queue.Add(*i); -/* typename Table::ConstIterator it; - meaningless ^= table.Find(*i, it);*/ } bool meaningless = queue.Drain(); - std::cout << entries << ' ' << size << ' ' << (inserted / static_cast<double>(entries)) << ' ' << (UserTime() - before_lookup) / static_cast<double>(queries_end - queries_begin) << '\n'; + std::cout << ' ' << (inserted / static_cast<double>(entries)) << ' ' << (CPUTime() - before_lookup) / static_cast<double>(queries_end - queries_begin) << std::flush; return meaningless; } -template <class Mod> bool TestRun(uint64_t lookups = 20000000, float multiplier = 1.5) { +bool TestRun(uint64_t lookups = 20000000, float multiplier = 1.5) { URandom rn; - util::scoped_malloc queries(util::CallocOrThrow(lookups * sizeof(uint64_t))); + util::scoped_memory queries; + HugeMalloc(lookups * sizeof(uint64_t), true, queries); rn.Batch(static_cast<uint64_t*>(queries.get()), static_cast<uint64_t*>(queries.get()) + lookups); uint64_t physical_mem_limit = util::GuessPhysicalMemory() / 2; bool meaningless = true; for (uint64_t i = 4; Size(i / multiplier) < physical_mem_limit; i *= 4) { - meaningless ^= util::Test<Mod>(rn, i / multiplier, static_cast<const uint64_t*>(queries.get()), static_cast<const uint64_t*>(queries.get()) + lookups, multiplier); + std::cout << static_cast<std::size_t>(i / multiplier) << ' ' << Size(i / multiplier); + typedef util::ProbingHashTable<Entry, util::IdentityHash, std::equal_to<Entry::Key>, Power2Mod> Table; + typedef util::ProbingHashTable<Entry, util::IdentityHash, std::equal_to<Entry::Key>, DivMod> TableDiv; + const uint64_t *const queries_begin = static_cast<const uint64_t*>(queries.get()); + meaningless ^= util::Test<Immediate<TableDiv> >(rn, i / multiplier, queries_begin, queries_begin + lookups, true, multiplier); + meaningless ^= util::Test<Immediate<Table> >(rn, i / multiplier, queries_begin, queries_begin + lookups, true, multiplier); + meaningless ^= util::Test<PrefetchQueue<Table, 4> >(rn, i / multiplier, queries_begin, queries_begin + lookups, true, multiplier); + meaningless ^= util::Test<Immediate<Table> >(rn, i / multiplier, queries_begin, queries_begin + lookups, false, multiplier); + meaningless ^= util::Test<PrefetchQueue<Table, 2> >(rn, i / multiplier, queries_begin, queries_begin + lookups, false, multiplier); + meaningless ^= util::Test<PrefetchQueue<Table, 4> >(rn, i / multiplier, queries_begin, queries_begin + lookups, false, multiplier); + meaningless ^= util::Test<PrefetchQueue<Table, 8> >(rn, i / multiplier, queries_begin, queries_begin + lookups, false, multiplier); + meaningless ^= util::Test<PrefetchQueue<Table, 16> >(rn, i / multiplier, queries_begin, queries_begin + lookups, false, multiplier); + std::cout << std::endl; } return meaningless; } @@ -148,9 +175,7 @@ template <class Mod> bool TestRun(uint64_t lookups = 20000000, float multiplier int main() { bool meaningless = false; - std::cout << "#Integer division\n"; - meaningless ^= util::TestRun<util::DivMod>(); - std::cout << "#Masking\n"; - meaningless ^= util::TestRun<util::Power2Mod>(); + std::cout << "#CPU time\n"; + meaningless ^= util::TestRun(); std::cerr << "Meaningless: " << meaningless << '\n'; } diff --git a/util/scoped.cc b/util/scoped.cc index 84f4344b7..817aa2424 100644 --- a/util/scoped.cc +++ b/util/scoped.cc @@ -27,7 +27,7 @@ void *MallocOrThrow(std::size_t requested) { } void *CallocOrThrow(std::size_t requested) { - return InspectAddr(std::calloc(1, requested), requested, "calloc"); + return InspectAddr(std::calloc(requested, 1), requested, "calloc"); } void scoped_malloc::call_realloc(std::size_t requested) { diff --git a/util/stream/rewindable_stream.cc b/util/stream/rewindable_stream.cc index 9421b25c3..726e2a72d 100644 --- a/util/stream/rewindable_stream.cc +++ b/util/stream/rewindable_stream.cc @@ -1,6 +1,5 @@ #include "util/stream/rewindable_stream.hh" #include "util/pcqueue.hh" -#include <iostream> #include <iostream> diff --git a/util/string_stream.hh b/util/string_stream.hh new file mode 100644 index 000000000..ee76a7a57 --- /dev/null +++ b/util/string_stream.hh @@ -0,0 +1,55 @@ +#ifndef UTIL_STRING_STREAM_H +#define UTIL_STRING_STREAM_H + +#include "util/fake_ostream.hh" + +#include <cassert> +#include <string> + +namespace util { + +class StringStream : public FakeOStream<StringStream> { + public: + // Semantics: appends to string. Remember to clear first! + + explicit StringStream() + {} + /* + explicit StringStream(std::string &out) + : out_(out) {} + */ + StringStream &flush() { return *this; } + + StringStream &write(const void *data, std::size_t length) { + out_.append(static_cast<const char*>(data), length); + return *this; + } + + const std::string &str() const + { return out_; } + void str(const std::string &val) + { + out_ = val; + } + + protected: + friend class FakeOStream<StringStream>; + char *Ensure(std::size_t amount) { + std::size_t current = out_.size(); + out_.resize(out_.size() + amount); + return &out_[current]; + } + + void AdvanceTo(char *to) { + assert(to <= &*out_.end()); + assert(to >= &*out_.begin()); + out_.resize(to - &*out_.begin()); + } + + private: + std::string out_; +}; + +} // namespace + +#endif // UTIL_STRING_STREAM_H diff --git a/util/string_stream_test.cc b/util/string_stream_test.cc new file mode 100644 index 000000000..ad996550c --- /dev/null +++ b/util/string_stream_test.cc @@ -0,0 +1,80 @@ +#define BOOST_LEXICAL_CAST_ASSUME_C_LOCALE +#define BOOST_TEST_MODULE FakeOStreamTest + +#include "util/string_stream.hh" +#include <boost/test/unit_test.hpp> +#include <boost/lexical_cast.hpp> + +#include <cstddef> +#include <limits> + +namespace util { namespace { + +template <class T> void TestEqual(const T value) { + StringStream strme; + strme << value; + BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), strme.str()); +} + +template <class T> void TestCorners() { + TestEqual(std::numeric_limits<T>::max()); + TestEqual(std::numeric_limits<T>::min()); + TestEqual(static_cast<T>(0)); + TestEqual(static_cast<T>(-1)); + TestEqual(static_cast<T>(1)); +} + +BOOST_AUTO_TEST_CASE(Integer) { + TestCorners<char>(); + TestCorners<signed char>(); + TestCorners<unsigned char>(); + + TestCorners<short>(); + TestCorners<signed short>(); + TestCorners<unsigned short>(); + + TestCorners<int>(); + TestCorners<unsigned int>(); + TestCorners<signed int>(); + + TestCorners<long>(); + TestCorners<unsigned long>(); + TestCorners<signed long>(); + + TestCorners<long long>(); + TestCorners<unsigned long long>(); + TestCorners<signed long long>(); + + TestCorners<std::size_t>(); +} + +enum TinyEnum { EnumValue }; + +BOOST_AUTO_TEST_CASE(EnumCase) { + TestEqual(EnumValue); +} + +BOOST_AUTO_TEST_CASE(Strings) { + TestEqual("foo"); + const char *a = "bar"; + TestEqual(a); + StringPiece piece("abcdef"); + TestEqual(piece); + TestEqual(StringPiece()); + + char non_const[3]; + non_const[0] = 'b'; + non_const[1] = 'c'; + non_const[2] = 0; + + StringStream out; + out << "a" << non_const << 'c'; + BOOST_CHECK_EQUAL("abcc", out.str()); + + // Now test as a separate object. + StringStream stream; + stream << "a" << non_const << 'c' << piece; + BOOST_CHECK_EQUAL("abccabcdef", stream.str()); +} + +}} // namespaces diff --git a/util/usage.cc b/util/usage.cc index 5f66b17d2..cfefe274e 100644 --- a/util/usage.cc +++ b/util/usage.cc @@ -135,14 +135,26 @@ double WallTime() { return Subtract(GetWall(), kRecordStart.Started()); } -double UserTime() { -#if !defined(_WIN32) && !defined(_WIN64) +double CPUTime() { +#if defined(_WIN32) || defined(_WIN64) + return 0.0; +#else struct rusage usage; if (getrusage(RUSAGE_SELF, &usage)) return 0.0; - return DoubleSec(usage.ru_utime); + return DoubleSec(usage.ru_utime) + DoubleSec(usage.ru_stime); +#endif +} + +uint64_t RSSMax() { +#if defined(_WIN32) || defined(_WIN64) + return 0; +#else + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) + return 0; + return static_cast<uint64_t>(usage.ru_maxrss) * 1024; #endif - return 0.0; } void PrintUsage(std::ostream &out) { @@ -274,6 +286,7 @@ template <class Num> uint64_t ParseNum(const std::string &arg) { return static_cast<uint64_t>(static_cast<double>(value) * static_cast<double>(mem) / 100.0); } + if (after == "k") after = "K"; std::string units("bKMGTPEZY"); std::string::size_type index = units.find(after[0]); UTIL_THROW_IF_ARG(index == std::string::npos, SizeParseError, (arg), "the allowed suffixes are " << units << "%."); diff --git a/util/usage.hh b/util/usage.hh index dff81b59d..2f1b3e992 100644 --- a/util/usage.hh +++ b/util/usage.hh @@ -9,7 +9,11 @@ namespace util { // Time in seconds since process started. Zero on unsupported platforms. double WallTime(); -double UserTime(); +// User + system time. +double CPUTime(); + +// Resident usage in bytes. +uint64_t RSSMax(); void PrintUsage(std::ostream &to); |