From 04fe2f4c25f1c8a4464f2d35be7166933dbb3aa0 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Wed, 2 Sep 2020 10:17:33 +0000 Subject: Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..70c8ab651 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,25 @@ +# Starter pipeline +- task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' -- cgit v1.2.3 From 53ea29008f1671a2ff955183a95d06e1cea850cf Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Wed, 2 Sep 2020 10:20:39 +0000 Subject: Updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 70c8ab651..35b8cc5d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,5 +1,5 @@ # Starter pipeline -- task: ComponentGovernanceComponentDetection@0 +task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' verbosity: 'Verbose' -- cgit v1.2.3 From fc1e484a8fada94e48d29445bdf1c5bbfc9c3f00 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Wed, 2 Sep 2020 10:24:07 +0000 Subject: Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 35b8cc5d7..b03473e29 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,10 +1,4 @@ # Starter pipeline -task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # Start with a minimal pipeline that you can customize to build and deploy your code. # Add steps that build, run tests, deploy, and more: # https://aka.ms/yaml @@ -23,3 +17,9 @@ steps: echo Add other tasks to build, test, and deploy your project. echo See https://aka.ms/yaml displayName: 'Run a multi-line script' + +- task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'LogOnly' + verbosity: 'Verbose' + alertWarningLevel: 'High' \ No newline at end of file -- cgit v1.2.3 From 5ab839aa3e8f02fc49205a0172b65fe3921b30e6 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Wed, 2 Sep 2020 10:28:05 +0000 Subject: Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b03473e29..0fa1a914a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,4 +22,4 @@ steps: inputs: scanType: 'LogOnly' verbosity: 'Verbose' - alertWarningLevel: 'High' \ No newline at end of file + alertWarningLevel: 'High' -- cgit v1.2.3 From 5f1c2c2d805f6b2bd2bd755604fe75f539092118 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Wed, 2 Sep 2020 10:28:44 +0000 Subject: Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0fa1a914a..7ad7a4d8f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,6 @@ steps: - task: ComponentGovernanceComponentDetection@0 inputs: - scanType: 'LogOnly' + scanType: 'Register' verbosity: 'Verbose' - alertWarningLevel: 'High' + alertWarningLevel: 'High' \ No newline at end of file -- cgit v1.2.3 From 33da1af73a76fa5083d98e0f8a95a33af53a49dc Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 12:55:22 +0530 Subject: added new pipelines --- azure-pipelines.yml | 81 ++++++++++++++++++++++++++++++++++++++++++++++++----- env-check.yml | 34 ++++++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 env-check.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7ad7a4d8f..d02280fcf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,19 +7,86 @@ trigger: - master pool: - vmImage: 'ubuntu-latest' + #vmImage: 'ubuntu-latest' + vmImage: ubuntu-16.04 steps: -- script: echo Hello, world! - displayName: 'Run a one-line script' - script: | - echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml - displayName: 'Run a multi-line script' + echo Printing some environment information + echo HOME: $HOME + echo + echo UBUNTU VERSION: + cat /etc/lsb-release + echo + echo CPU INFO + cat /proc/cpuinfo + echo + echo MEM INFO + cat /proc/meminfo + echo + echo DISK INFO + df -h + echo + echo PWD: $PWD + echo + ls + displayName: 'Printing some environment information' + + +## Installation commands for Ubuntu +- script: | + sudo apt-get install \ + g++ \ + git \ + subversion \ + automake \ + libtool \ + zlib1g-dev \ + libicu-dev \ + libboost-all-dev \ + libbz2-dev \ + liblzma-dev \ + python-dev \ + graphviz \ + imagemagick \ + make \ + cmake \ + libgoogle-perftools-dev (for tcmalloc) \ + autoconf \ + doxygen + displayName: 'Install Ubuntu packages' + +- script: | + wget https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download + tar xvzf cmph-2.0.2.tar.gz + cd cmph-2.0.2.tar.gz + ./configure --prefix=$PWD + make + make install + cd .. + displayName: 'Build and Install cmph' + +- script: | + wget https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download + tar xvzf xmlrpc-c-1.51.06.tgz + cd xmlrpc-c-1.51.06.tgz + ./configure --prefix=$PWD + make + make install + ldconfig + cd .. + displayName: 'Build and Install xmlrpc-c' + +- script: | + ./bjam \ + --with-cmph=$PWD/cmph-2.0.2 \ + --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ + -j3 + displayName: 'Build Moses' - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' verbosity: 'Verbose' - alertWarningLevel: 'High' \ No newline at end of file + alertWarningLevel: 'High' \ No newline at end of file diff --git a/env-check.yml b/env-check.yml new file mode 100644 index 000000000..43c23fb52 --- /dev/null +++ b/env-check.yml @@ -0,0 +1,34 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + #vmImage: 'ubuntu-latest' + vmImage: ubuntu-16.04 + +steps: + +- script: | + echo Printing some environment information + echo HOME: $HOME + echo + echo UBUNTU VERSION: + cat /etc/lsb-release + echo + echo CPU INFO + cat /proc/cpuinfo + echo + echo MEM INFO + cat /proc/meminfo + echo + echo DISK INFO + df -h + echo + echo PWD: $PWD + echo + ls + displayName: 'Printing some environment information' \ No newline at end of file -- cgit v1.2.3 From 613a8e6bc78116fa9eb019042eda4247da0f5af3 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 07:27:47 +0000 Subject: Set up CI with Azure Pipelines correct image [skip ci] --- env-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env-check.yml b/env-check.yml index 43c23fb52..9292648fa 100644 --- a/env-check.yml +++ b/env-check.yml @@ -8,7 +8,7 @@ trigger: pool: #vmImage: 'ubuntu-latest' - vmImage: ubuntu-16.04 + vmImage: 'ubuntu-16.04' steps: -- cgit v1.2.3 From 70aebfcbfb730d72bcca28fd676a881b0f47a80a Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 13:04:53 +0530 Subject: fix errors in pipeline --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d02280fcf..05aa036d0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,7 @@ trigger: pool: #vmImage: 'ubuntu-latest' - vmImage: ubuntu-16.04 + vmImage: 'ubuntu-16.04' steps: @@ -52,7 +52,7 @@ steps: imagemagick \ make \ cmake \ - libgoogle-perftools-dev (for tcmalloc) \ + libgoogle-perftools-dev \ autoconf \ doxygen displayName: 'Install Ubuntu packages' @@ -82,7 +82,7 @@ steps: ./bjam \ --with-cmph=$PWD/cmph-2.0.2 \ --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ - -j3 + -j2 displayName: 'Build Moses' - task: ComponentGovernanceComponentDetection@0 -- cgit v1.2.3 From 7b4becfcddaa2b492605aa12f238de0817380c4f Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 13:17:17 +0530 Subject: fix errors in download of libraries --- azure-pipelines.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 05aa036d0..c1e5fe375 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,9 +58,10 @@ steps: displayName: 'Install Ubuntu packages' - script: | - wget https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download + wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" + mv download cmph-2.0.2.tar.gz tar xvzf cmph-2.0.2.tar.gz - cd cmph-2.0.2.tar.gz + cd cmph-2.0.2 ./configure --prefix=$PWD make make install @@ -69,8 +70,9 @@ steps: - script: | wget https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download + mv download xmlrpc-c-1.51.06.tgz tar xvzf xmlrpc-c-1.51.06.tgz - cd xmlrpc-c-1.51.06.tgz + cd xmlrpc-c-1.51.06 ./configure --prefix=$PWD make make install -- cgit v1.2.3 From 70b1a28d70a4d33c57aaf5ac82b593d9b0ebafad Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 13:18:46 +0530 Subject: fix build errors --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c1e5fe375..e6d6785ae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -69,7 +69,7 @@ steps: displayName: 'Build and Install cmph' - script: | - wget https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download + wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" mv download xmlrpc-c-1.51.06.tgz tar xvzf xmlrpc-c-1.51.06.tgz cd xmlrpc-c-1.51.06 -- cgit v1.2.3 From e9341518459b3c41443484cc0e58d867d9c663e4 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 13:54:43 +0530 Subject: changes to xmlrpc build and link --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e6d6785ae..7998f0f76 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ steps: ./configure --prefix=$PWD make make install - ldconfig + sudo ldconfig cd .. displayName: 'Build and Install xmlrpc-c' -- cgit v1.2.3 From 1ec64c8816aacbfb930f6e27b845ed035284a1a1 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 14:24:46 +0530 Subject: add libssl-dev package installation to build pipeline --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7998f0f76..123b94891 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -45,6 +45,7 @@ steps: zlib1g-dev \ libicu-dev \ libboost-all-dev \ + libssl-dev \ libbz2-dev \ liblzma-dev \ python-dev \ -- cgit v1.2.3 From 0cc1727fb426e5b137f17593b2dd37a12b6d0998 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 14:40:32 +0530 Subject: simplify moses build (no xmplrpc and cmph) --- azure-pipelines.yml | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 123b94891..c00c7d3b4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,35 +58,40 @@ steps: doxygen displayName: 'Install Ubuntu packages' -- script: | - wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" - mv download cmph-2.0.2.tar.gz - tar xvzf cmph-2.0.2.tar.gz - cd cmph-2.0.2 - ./configure --prefix=$PWD - make - make install - cd .. - displayName: 'Build and Install cmph' +# - script: | +# wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" +# mv download cmph-2.0.2.tar.gz +# tar xvzf cmph-2.0.2.tar.gz +# cd cmph-2.0.2 +# ./configure --prefix=$PWD +# make +# make install +# cd .. +# displayName: 'Build and Install cmph' -- script: | - wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" - mv download xmlrpc-c-1.51.06.tgz - tar xvzf xmlrpc-c-1.51.06.tgz - cd xmlrpc-c-1.51.06 - ./configure --prefix=$PWD - make - make install - sudo ldconfig - cd .. - displayName: 'Build and Install xmlrpc-c' +# - script: | +# wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" +# mv download xmlrpc-c-1.51.06.tgz +# tar xvzf xmlrpc-c-1.51.06.tgz +# cd xmlrpc-c-1.51.06 +# ./configure --prefix=$PWD +# make +# make install +# sudo ldconfig +# cd .. +# displayName: 'Build and Install xmlrpc-c' + +# - script: | +# ./bjam \ +# --with-cmph=$PWD/cmph-2.0.2 \ +# --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ +# -j2 +# displayName: 'Build Moses' - script: | ./bjam \ - --with-cmph=$PWD/cmph-2.0.2 \ - --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ -j2 - displayName: 'Build Moses' + displayName: 'Build Moses' - task: ComponentGovernanceComponentDetection@0 inputs: -- cgit v1.2.3 From 1d2424cd6813a3b96f463e0a15ac436b1784e2c3 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 09:38:42 +0000 Subject: Remove explicit component governance task --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c00c7d3b4..8eb153e35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -93,8 +93,8 @@ steps: -j2 displayName: 'Build Moses' -- task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' \ No newline at end of file +# - task: ComponentGovernanceComponentDetection@0 +# inputs: +# scanType: 'Register' +# verbosity: 'Verbose' +# alertWarningLevel: 'High' \ No newline at end of file -- cgit v1.2.3 From f15249b14b3d3c30438f165e374107e69011ba0f Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 16:57:11 +0530 Subject: added cgmanifest file --- cgmanifest.json | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 cgmanifest.json diff --git a/cgmanifest.json b/cgmanifest.json new file mode 100644 index 000000000..91e211847 --- /dev/null +++ b/cgmanifest.json @@ -0,0 +1,33 @@ +{ + "Registrations":[ + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://github.com/moses-smt/mosesdecoder", + "commitHash": "78ca5f3cc5aa671a8a5d36c56452e217e6f00828" + } + } + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "cmph", + "Version": "2.0.2", + "DownloadUrl": "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" + } + } + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "xml-rpc-c", + "Version": "1.51.06", + "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" + } + } + }, + ] +} \ No newline at end of file -- cgit v1.2.3 From 8fbb193b2bc564151a060855193d57283ba6a595 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 18:00:48 +0530 Subject: added cmph information --- cgmanifest.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cgmanifest.json b/cgmanifest.json index 91e211847..eb3008ba5 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -9,16 +9,15 @@ } } }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "cmph", - "Version": "2.0.2", - "DownloadUrl": "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" - } - } - }, + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://git.code.sf.net/p/cmph/git", + "commitHash": "a250982ade093f4eed0552bbdd22dd7b0432007f" + } + } + }, { "Component": { "Type": "other", @@ -30,4 +29,5 @@ } }, ] -} \ No newline at end of file +} + -- cgit v1.2.3 From 2b6d72ccb954d4d87f8b347c2b09b2b05f4fdc15 Mon Sep 17 00:00:00 2001 From: "Anoop Kunchukuttan (STC INDIA)" Date: Thu, 3 Sep 2020 18:07:24 +0530 Subject: corrected URL for cmph --- cgmanifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgmanifest.json b/cgmanifest.json index eb3008ba5..6e7a9eafb 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -24,7 +24,7 @@ "Other": { "Name": "xml-rpc-c", "Version": "1.51.06", - "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" + "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz" } } }, -- cgit v1.2.3 From c635efaf2385b2fe692501543fe6a8ab26c87ef9 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Wed, 30 Sep 2020 21:34:53 +0530 Subject: added to typedef --- moses2/TypeDef.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moses2/TypeDef.h b/moses2/TypeDef.h index aed39ac11..d96257ac2 100644 --- a/moses2/TypeDef.h +++ b/moses2/TypeDef.h @@ -71,7 +71,8 @@ enum InputTypeEnum { TreeInputType = 3, //,WordLatticeInput2 = 4, TabbedSentenceInput = 5, - ForestInputType = 6 + ForestInputType = 6, + SentenceInputWithCandidates = 7, }; enum XmlInputType { -- cgit v1.2.3 From 2fa5c1cfad310b0be7a656167a7bd54b05baba4a Mon Sep 17 00:00:00 2001 From: sid jain Date: Wed, 30 Sep 2020 21:38:43 +0530 Subject: sample --- chk.tmp | 1 + 1 file changed, 1 insertion(+) create mode 100644 chk.tmp diff --git a/chk.tmp b/chk.tmp new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/chk.tmp @@ -0,0 +1 @@ +test -- cgit v1.2.3 From a6c60a64125f61133a7efd51a3c789c7384bc821 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Wed, 30 Sep 2020 22:19:48 +0530 Subject: add SentenceWithCandidates class --- moses2/PhraseBased/Manager.cpp | 2 +- moses2/PhraseBased/SentenceWithCandidates.cpp | 185 ++++++++++++++++++++++++++ moses2/PhraseBased/SentenceWithCandidates.h | 52 ++++++++ 3 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 moses2/PhraseBased/SentenceWithCandidates.cpp create mode 100644 moses2/PhraseBased/SentenceWithCandidates.h diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index 28073d4f6..670c33894 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -59,7 +59,7 @@ void Manager::Init() InitPools(); FactorCollection &vocab = system.GetVocab(); - m_input = Moses2::Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr); + m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); m_bitmaps = new Bitmaps(GetPool()); diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp new file mode 100644 index 000000000..4f55c5578 --- /dev/null +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -0,0 +1,185 @@ +/* + * SentenceWithCandidates.cpp + * + * Created on: 14 Dec 2015 + * Author: hieu + */ +#include +#include +#include + +#include "SentenceWithCandidates.h" +#include "../System.h" +#include "../parameters/AllOptions.h" +#include "../legacy/Util2.h" + + +using namespace std; + +namespace Moses2 +{ + +SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, FactorCollection &vocab, + const System &system, const std::string &str) +{ + SentenceWithCandidates *ret; + + vector result; + boost::split(result, str, boost::is_any_of("|||")); + + if (result.size()!=2){ + exit(1); + } + + const string partstr = result[0] + + if (system.options.input.xml_policy) { + // xml + ret = CreateFromStringXML(pool, vocab, system, partstr); + } else { + // no xml + //cerr << "PB SentenceWithCandidates" << endl; + std::vector toks = Tokenize(partstr); + + size_t size = toks.size(); + ret = new (pool.Allocate()) SentenceWithCandidates(pool, size); + ret->PhraseImplTemplate::CreateFromString(vocab, system, toks, false); + } + + //cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl; + //cerr << "ret=" << ret->Debug(system) << endl; + + return ret; +} + +SentenceWithCandidates *SentenceWithCandidates::CreateFromStringXML(MemPool &pool, FactorCollection &vocab, + const System &system, const std::string &str) +{ + SentenceWithCandidates *ret; + + vector xmlOptions; + pugi::xml_document doc; + + string str2 = "" + str + ""; + pugi::xml_parse_result result = doc.load(str2.c_str(), + pugi::parse_cdata | pugi::parse_wconv_attribute | pugi::parse_eol | pugi::parse_comments); + pugi::xml_node topNode = doc.child("xml"); + + std::vector toks; + XMLParse(pool, system, 0, topNode, toks, xmlOptions); + + // debug + /* + cerr << "xmloptions:" << endl; + for (size_t i = 0; i < xmlOptions.size(); ++i) { + cerr << xmlOptions[i]->Debug(system) << endl; + } + */ + + // create words + size_t size = toks.size(); + ret = new (pool.Allocate()) SentenceWithCandidates(pool, size); + ret->PhraseImplTemplate::CreateFromString(vocab, system, toks, false); + + // xml + ret->Init(system, size, system.options.reordering.max_distortion); + + ReorderingConstraint &reorderingConstraint = ret->GetReorderingConstraint(); + + // set reordering walls, if "-monotone-at-punction" is set + if (system.options.reordering.monotone_at_punct && ret->GetSize()) { + reorderingConstraint.SetMonotoneAtPunctuation(*ret); + } + + // set walls obtained from xml + for(size_t i=0; iGetNodeName(), "wall") == 0) { + if (xmlOption->startPos) { + UTIL_THROW_IF2(xmlOption->startPos > ret->GetSize(), "wall is beyond the SentenceWithCandidates"); // no buggy walls, please + reorderingConstraint.SetWall(xmlOption->startPos - 1, true); + } + } else if (strcmp(xmlOption->GetNodeName(), "zone") == 0) { + reorderingConstraint.SetZone( xmlOption->startPos, xmlOption->startPos + xmlOption->phraseSize -1 ); + } else if (strcmp(xmlOption->GetNodeName(), "ne") == 0) { + FactorType placeholderFactor = system.options.input.placeholder_factor; + UTIL_THROW_IF2(placeholderFactor == NOT_FOUND, + "Placeholder XML in input. Must have argument -placeholder-factor [NUM]"); + UTIL_THROW_IF2(xmlOption->phraseSize != 1, + "Placeholder must only cover 1 word"); + + const Factor *factor = vocab.AddFactor(xmlOption->GetEntity(), system, false); + (*ret)[xmlOption->startPos][placeholderFactor] = factor; + } else { + // default - forced translation. Add to class variable + ret->AddXMLOption(system, xmlOption); + } + } + reorderingConstraint.FinalizeWalls(); + + return ret; +} + +void SentenceWithCandidates::XMLParse( + MemPool &pool, + const System &system, + size_t depth, + const pugi::xml_node &parentNode, + std::vector &toks, + vector &xmlOptions) +{ + // pugixml + for (pugi::xml_node childNode = parentNode.first_child(); childNode; childNode = childNode.next_sibling()) { + string nodeName = childNode.name(); + //cerr << depth << " nodeName=" << nodeName << endl; + + int startPos = toks.size(); + + string value = childNode.value(); + if (!value.empty()) { + //cerr << depth << "childNode text=" << value << endl; + std::vector subPhraseToks = Tokenize(value); + for (size_t i = 0; i < subPhraseToks.size(); ++i) { + toks.push_back(subPhraseToks[i]); + } + } + + if (!nodeName.empty()) { + XMLOption *xmlOption = new (pool.Allocate()) XMLOption(pool, nodeName, startPos); + + pugi::xml_attribute attr; + attr = childNode.attribute("translation"); + if (!attr.empty()) { + xmlOption->SetTranslation(pool, attr.as_string()); + } + + attr = childNode.attribute("entity"); + if (!attr.empty()) { + xmlOption->SetEntity(pool, attr.as_string()); + } + + attr = childNode.attribute("prob"); + if (!attr.empty()) { + xmlOption->prob = attr.as_float(); + } + + xmlOptions.push_back(xmlOption); + + // recursively call this function. For proper recursive trees + XMLParse(pool, system, depth + 1, childNode, toks, xmlOptions); + + size_t endPos = toks.size(); + xmlOption->phraseSize = endPos - startPos; + + /* + cerr << "xmlOptions="; + xmlOption->Debug(cerr, system); + cerr << endl; + */ + } + + } +} + +} /* namespace Moses2 */ + diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h new file mode 100644 index 000000000..6cfea1a22 --- /dev/null +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -0,0 +1,52 @@ +/* + * SentenceWithCandidates.h + * + * Created on: 14 Dec 2015 + * Author: hieu + */ +#pragma once + +#include +#include +#include "PhraseImpl.h" +#include "../InputType.h" +#include "../MemPool.h" +#include "../pugixml.hpp" +#include "../legacy/Util2.h" + +namespace Moses2 +{ +class FactorCollection; +class System; + +class SentenceWithCandidates: public InputType, public PhraseImpl +{ +public: + + static SentenceWithCandidates *CreateFromString(MemPool &pool, FactorCollection &vocab, + const System &system, const std::string &str); + + SentenceWithCandidates(MemPool &pool, size_t size) + :InputType(pool) + ,PhraseImpl(pool, size) + {} + + virtual ~SentenceWithCandidates() + {} + +protected: + static SentenceWithCandidates *CreateFromStringXML(MemPool &pool, FactorCollection &vocab, + const System &system, const std::string &str); + + static void XMLParse( + MemPool &pool, + const System &system, + size_t depth, + const pugi::xml_node &parentNode, + std::vector &toks, + std::vector &xmlOptions); + +}; + +} /* namespace Moses2 */ + -- cgit v1.2.3 From d325b0d274c4e29ed632759f67c7a51642b0d6c1 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Wed, 30 Sep 2020 22:38:27 +0530 Subject: restored build commands --- azure-pipelines.yml | 58 ++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8eb153e35..fddd0faea 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,40 +58,40 @@ steps: doxygen displayName: 'Install Ubuntu packages' -# - script: | -# wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" -# mv download cmph-2.0.2.tar.gz -# tar xvzf cmph-2.0.2.tar.gz -# cd cmph-2.0.2 -# ./configure --prefix=$PWD -# make -# make install -# cd .. -# displayName: 'Build and Install cmph' - -# - script: | -# wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" -# mv download xmlrpc-c-1.51.06.tgz -# tar xvzf xmlrpc-c-1.51.06.tgz -# cd xmlrpc-c-1.51.06 -# ./configure --prefix=$PWD -# make -# make install -# sudo ldconfig -# cd .. -# displayName: 'Build and Install xmlrpc-c' +- script: | + wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download" + mv download cmph-2.0.2.tar.gz + tar xvzf cmph-2.0.2.tar.gz + cd cmph-2.0.2 + ./configure --prefix=$PWD + make + make install + cd .. + displayName: 'Build and Install cmph' -# - script: | -# ./bjam \ -# --with-cmph=$PWD/cmph-2.0.2 \ -# --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ -# -j2 -# displayName: 'Build Moses' +- script: | + wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download" + mv download xmlrpc-c-1.51.06.tgz + tar xvzf xmlrpc-c-1.51.06.tgz + cd xmlrpc-c-1.51.06 + ./configure --prefix=$PWD + make + make install + sudo ldconfig + cd .. + displayName: 'Build and Install xmlrpc-c' - script: | ./bjam \ + --with-cmph=$PWD/cmph-2.0.2 \ + --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \ -j2 - displayName: 'Build Moses' + displayName: 'Build Moses' + +# - script: | +# ./bjam \ +# -j2 +# displayName: 'Build Moses' # - task: ComponentGovernanceComponentDetection@0 # inputs: -- cgit v1.2.3 From ad829d3b3fc718955a2c6d27a123b17400cd3c1b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 30 Sep 2020 10:35:08 -0700 Subject: compiles --- moses2/Jamfile | 3 +- moses2/PhraseBased/Manager.cpp | 1 + moses2/PhraseBased/SentenceWithCandidates.cpp | 2 +- moses2/PhraseBased/SentenceWithCandidates.h | 7 +- moses2/TranslationModel/MSPT/MSNode.h | 131 +++++++++++++ moses2/TranslationModel/MSPT/MSPT.cpp | 265 ++++++++++++++++++++++++++ moses2/TranslationModel/MSPT/MSPT.h | 85 +++++++++ 7 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 moses2/TranslationModel/MSPT/MSNode.h create mode 100644 moses2/TranslationModel/MSPT/MSPT.cpp create mode 100644 moses2/TranslationModel/MSPT/MSPT.h diff --git a/moses2/Jamfile b/moses2/Jamfile index 42676c065..e8a6457fd 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -108,7 +108,8 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors PhraseBased/ReorderingConstraint.cpp PhraseBased/TargetPhrases.cpp PhraseBased/Search.cpp - PhraseBased/Sentence.cpp + PhraseBased/Sentence.cpp + PhraseBased/SentenceWithCandidates.cpp PhraseBased/TargetPhraseImpl.cpp PhraseBased/TrellisPath.cpp diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index 670c33894..87620e853 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -13,6 +13,7 @@ #include "TargetPhraseImpl.h" #include "InputPath.h" #include "Sentence.h" +#include "SentenceWithCandidates.h" #include "Normal/Search.h" #include "CubePruningMiniStack/Search.h" diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index 4f55c5578..15e8294f3 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -31,7 +31,7 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, exit(1); } - const string partstr = result[0] + const string partstr = result[0]; if (system.options.input.xml_policy) { // xml diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 6cfea1a22..8148ce8f3 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -9,7 +9,7 @@ #include #include #include "PhraseImpl.h" -#include "../InputType.h" +#include "Sentence.h" #include "../MemPool.h" #include "../pugixml.hpp" #include "../legacy/Util2.h" @@ -19,7 +19,7 @@ namespace Moses2 class FactorCollection; class System; -class SentenceWithCandidates: public InputType, public PhraseImpl +class SentenceWithCandidates: public Sentence { public: @@ -27,8 +27,7 @@ public: const System &system, const std::string &str); SentenceWithCandidates(MemPool &pool, size_t size) - :InputType(pool) - ,PhraseImpl(pool, size) + :Sentence(pool, size) {} virtual ~SentenceWithCandidates() diff --git a/moses2/TranslationModel/MSPT/MSNode.h b/moses2/TranslationModel/MSPT/MSNode.h new file mode 100644 index 000000000..ad6d0842d --- /dev/null +++ b/moses2/TranslationModel/MSPT/MSNode.h @@ -0,0 +1,131 @@ +/* + * Node.h + * + * Created on: 22 Apr 2016 + * Author: hieu + */ +#pragma once +#include +#include +#include "../../PhraseBased/TargetPhrases.h" +#include "../../System.h" +#include "../../Phrase.h" + +namespace Moses2 +{ +class System; + +namespace MSPTNS +{ + +template +class Node +{ +public: + typedef boost::unordered_map Children; + + Node() + :m_targetPhrases(NULL) + ,m_unsortedTPS(NULL) + {} + + ~Node() + {} + + void AddRule(const std::vector &factors, SP &source, TP *target) { + AddRule(factors, source, target, 0); + } + + TPS *Find(const std::vector &factors, const SP &source, size_t pos = 0) const { + assert(source.GetSize()); + if (pos == source.GetSize()) { + return m_targetPhrases; + } else { + const WORD &word = source[pos]; + //cerr << "word=" << word << endl; + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } else { + const Node &child = iter->second; + return child.Find(factors, source, pos + 1); + } + } + } + + const Node *Find(const std::vector &factors, const WORD &word) const { + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } else { + const Node &child = iter->second; + return &child; + } + } + + const TPS *GetTargetPhrases() const { + return m_targetPhrases; + } + + void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) { + BOOST_FOREACH(typename Children::value_type &val, m_children) { + Node &child = val.second; + child.SortAndPrune(tableLimit, pool, system); + } + + // prune target phrases in this node + if (m_unsortedTPS) { + m_targetPhrases = new (pool.Allocate()) TPS(pool, m_unsortedTPS->size()); + + for (size_t i = 0; i < m_unsortedTPS->size(); ++i) { + TP *tp = (*m_unsortedTPS)[i]; + m_targetPhrases->AddTargetPhrase(*tp); + } + + m_targetPhrases->SortAndPrune(tableLimit); + system.featureFunctions.EvaluateAfterTablePruning(system.GetSystemPool(), *m_targetPhrases, *m_source); + + delete m_unsortedTPS; + } + } + + const Children &GetChildren() const { + return m_children; + } + + void Debug(std::ostream &out, const System &system) const { + BOOST_FOREACH(const typename Children::value_type &valPair, m_children) { + const WORD &word = valPair.first; + //std::cerr << word << "(" << word.hash() << ") "; + } + } +protected: + Children m_children; + TPS *m_targetPhrases; + Phrase *m_source; + std::vector *m_unsortedTPS; + + Node &AddRule(const std::vector &factors, SP &source, TP *target, size_t pos) { + if (pos == source.GetSize()) { + if (m_unsortedTPS == NULL) { + m_unsortedTPS = new std::vector(); + m_source = &source; + } + + m_unsortedTPS->push_back(target); + return *this; + } else { + const WORD &word = source[pos]; + Node &child = m_children[word.hash(factors)]; + //std::cerr << "added " << word << " " << &child << " from " << this << std::endl; + + return child.AddRule(factors, source, target, pos + 1); + } + } + +}; + + +} +} // namespace + diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp new file mode 100644 index 000000000..c905d5240 --- /dev/null +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -0,0 +1,265 @@ +/* + * MSPT.cpp + * + * Created on: 28 Oct 2015 + * Author: hieu + */ + +#include +#include +#include "MSPT.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../Phrase.h" +#include "../../System.h" +#include "../../Scores.h" +#include "../../InputPathsBase.h" +#include "../../legacy/InputFileStream.h" +#include "util/exception.hh" + +#include "../../PhraseBased/InputPath.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" + +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/InputPath.h" +#include "../../SCFG/Stack.h" +#include "../../SCFG/Stacks.h" +#include "../../SCFG/Manager.h" + + +using namespace std; + +namespace Moses2 +{ + + +//////////////////////////////////////////////////////////////////////// + +MSPT::MSPT(size_t startInd, const std::string &line) + :PhraseTable(startInd, line) + ,m_rootPb(NULL) + ,m_rootSCFG(NULL) +{ + ReadParameters(); +} + +MSPT::~MSPT() +{ + delete m_rootPb; + delete m_rootSCFG; +} + +void MSPT::Load(System &system) +{ + FactorCollection &vocab = system.GetVocab(); + MemPool &systemPool = system.GetSystemPool(); + MemPool tmpSourcePool; + + if (system.isPb) { + m_rootPb = new PBNODE(); + } else { + m_rootSCFG = new SCFGNODE(); + //cerr << "m_rootSCFG=" << m_rootSCFG << endl; + } + + vector toks; + size_t lineNum = 0; + InputFileStream strme(m_path); + string line; + while (getline(strme, line)) { + if (++lineNum % 1000000 == 0) { + cerr << lineNum << " "; + } + toks.clear(); + TokenizeMultiCharSeparator(toks, line, "|||"); + UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); + //cerr << "line=" << line << endl; + //cerr << "system.isPb=" << system.isPb << endl; + + if (system.isPb) { + PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created soure" << endl; + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, + toks[1]); + //cerr << "created target" << endl; + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + if (toks.size() >= 4) { + //cerr << "alignstr=" << toks[3] << endl; + target->SetAlignmentInfo(toks[3]); + } + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootPb->AddRule(m_input, *source, target); + + //cerr << "target=" << target->Debug(system) << endl; + } else { + SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created source:" << *source << endl; + SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, + system, toks[1]); + + //cerr << "created target " << *target << " source=" << *source << endl; + + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + //vector scores = Tokenize(toks[2]); + //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; + + target->SetAlignmentInfo(toks[3]); + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootSCFG->AddRule(m_input, *source, target); + } + } + + if (system.isPb) { + m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } else { + m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } + /* + BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { + const Word &word = valPair.first; + cerr << word << " "; + } + cerr << endl; + */ +} + +TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + const SubPhrase &phrase = inputPath.subPhrase; + TargetPhrases *tps = m_rootPb->Find(m_input, phrase); + return tps; +} + +void MSPT::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + size_t ptInd = GetPtInd(); + ActiveChartEntryMem *chartEntry = new (pool.Allocate()) ActiveChartEntryMem(pool, *m_rootSCFG); + path.AddActiveChartEntry(ptInd, chartEntry); + //cerr << "InitActiveChart=" << path << endl; +} + +void MSPT::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + if (path.range.GetNumWordsCovered() > maxChartSpan) { + return; + } + + size_t endPos = path.range.GetEndPos(); + + const SCFG::InputPath *prevPath = static_cast(path.prefixPath); + UTIL_THROW_IF2(prevPath == NULL, "prefixPath == NULL"); + + // TERMINAL + const SCFG::Word &lastWord = path.subPhrase.Back(); + + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1); + + //cerr << "BEFORE LookupGivenWord=" << *prevPath << endl; + LookupGivenWord(pool, mgr, *prevPath, lastWord, NULL, subPhrasePath.range, path); + //cerr << "AFTER LookupGivenWord=" << *prevPath << endl; + + // NON-TERMINAL + //const SCFG::InputPath *prefixPath = static_cast(path.prefixPath); + while (prevPath) { + const Range &prevRange = prevPath->range; + //cerr << "prevRange=" << prevRange << endl; + + size_t startPos = prevRange.GetEndPos() + 1; + size_t ntSize = endPos - startPos + 1; + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(startPos, ntSize); + + LookupNT(pool, mgr, subPhrasePath.range, *prevPath, stacks, path); + + prevPath = static_cast(prevPath->prefixPath); + } +} + +void MSPT::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + const ActiveChartEntryMem &prevEntryCast = static_cast(prevEntry); + + const SCFGNODE &prevNode = prevEntryCast.node; + UTIL_THROW_IF2(&prevNode == NULL, "node == NULL"); + + size_t ptInd = GetPtInd(); + const SCFGNODE *nextNode = prevNode.Find(m_input, wordSought); + + /* + if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) { + cerr << "range=" << outPath.range + << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) + << " wordSought=" << wordSought.Debug(mgr.system) + << " nextNode=" << nextNode + << endl; + } + */ + if (nextNode) { + // new entries + ActiveChartEntryMem *chartEntry = new (pool.Allocate()) ActiveChartEntryMem(pool, *nextNode, prevEntry); + + chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); + //cerr << "AFTER Add=" << symbolBind << endl; + + outPath.AddActiveChartEntry(ptInd, chartEntry); + + const SCFG::TargetPhrases *tps = nextNode->GetTargetPhrases(); + if (tps) { + // there are some rules + /* + cerr << "outPath=" << outPath.range + << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system) + << " pt=" << GetPtInd() + << " tps=" << tps->Debug(mgr.system) << endl; + */ + outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); + + } + + //cerr << "AFTER outPath=" << outPath << endl; + } +} + +} + diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h new file mode 100644 index 000000000..d3946d353 --- /dev/null +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -0,0 +1,85 @@ +/* + * MSPT.h + * + * Created on: 28 Oct 2015 + * Author: hieu + */ +#pragma once + +#include "../PhraseTable.h" +#include "../../legacy/Util2.h" +#include "../../SCFG/InputPath.h" +#include "MSNode.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/TargetPhrases.h" + +namespace Moses2 +{ + +class MSPT: public PhraseTable +{ + typedef MSPTNS::Node, TargetPhraseImpl, TargetPhrases> PBNODE; + typedef MSPTNS::Node, SCFG::TargetPhraseImpl, SCFG::TargetPhrases> SCFGNODE; + +////////////////////////////////////// + class ActiveChartEntryMem : public SCFG::ActiveChartEntry + { + typedef SCFG::ActiveChartEntry Parent; + public: + const MSPT::SCFGNODE &node; + + ActiveChartEntryMem(MemPool &pool, const MSPT::SCFGNODE &vnode) + :Parent(pool) + ,node(vnode) + {} + + ActiveChartEntryMem( + MemPool &pool, + const MSPT::SCFGNODE &vnode, + const ActiveChartEntry &prevEntry) + :Parent(prevEntry) + ,node(vnode) + {} + }; + + ////////////////////////////////////// +public: + MSPT(size_t startInd, const std::string &line); + virtual ~MSPT(); + + virtual void Load(System &system); + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + PBNODE *m_rootPb; + SCFGNODE *m_rootSCFG; + + void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + +}; + +} + -- cgit v1.2.3 From d74ef345918881919e57dca447d15c9fca4e5082 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 30 Sep 2020 11:06:38 -0700 Subject: delete SentenceWithCandidates::XMLParse --- moses2/PhraseBased/SentenceWithCandidates.cpp | 61 --------------------------- moses2/PhraseBased/SentenceWithCandidates.h | 8 ---- 2 files changed, 69 deletions(-) diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index 15e8294f3..f2703f7dd 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -120,66 +120,5 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromStringXML(MemPool &poo return ret; } -void SentenceWithCandidates::XMLParse( - MemPool &pool, - const System &system, - size_t depth, - const pugi::xml_node &parentNode, - std::vector &toks, - vector &xmlOptions) -{ - // pugixml - for (pugi::xml_node childNode = parentNode.first_child(); childNode; childNode = childNode.next_sibling()) { - string nodeName = childNode.name(); - //cerr << depth << " nodeName=" << nodeName << endl; - - int startPos = toks.size(); - - string value = childNode.value(); - if (!value.empty()) { - //cerr << depth << "childNode text=" << value << endl; - std::vector subPhraseToks = Tokenize(value); - for (size_t i = 0; i < subPhraseToks.size(); ++i) { - toks.push_back(subPhraseToks[i]); - } - } - - if (!nodeName.empty()) { - XMLOption *xmlOption = new (pool.Allocate()) XMLOption(pool, nodeName, startPos); - - pugi::xml_attribute attr; - attr = childNode.attribute("translation"); - if (!attr.empty()) { - xmlOption->SetTranslation(pool, attr.as_string()); - } - - attr = childNode.attribute("entity"); - if (!attr.empty()) { - xmlOption->SetEntity(pool, attr.as_string()); - } - - attr = childNode.attribute("prob"); - if (!attr.empty()) { - xmlOption->prob = attr.as_float(); - } - - xmlOptions.push_back(xmlOption); - - // recursively call this function. For proper recursive trees - XMLParse(pool, system, depth + 1, childNode, toks, xmlOptions); - - size_t endPos = toks.size(); - xmlOption->phraseSize = endPos - startPos; - - /* - cerr << "xmlOptions="; - xmlOption->Debug(cerr, system); - cerr << endl; - */ - } - - } -} - } /* namespace Moses2 */ diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 8148ce8f3..4dafca4af 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -37,14 +37,6 @@ protected: static SentenceWithCandidates *CreateFromStringXML(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str); - static void XMLParse( - MemPool &pool, - const System &system, - size_t depth, - const pugi::xml_node &parentNode, - std::vector &toks, - std::vector &xmlOptions); - }; } /* namespace Moses2 */ -- cgit v1.2.3 From d97c54b7290b7aced558f3aae569e6c6dce7ce1f Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Thu, 1 Oct 2020 08:38:30 +0530 Subject: changes to handle part input --- cgmanifest.json | 66 +++++++++++++-------------- moses2/PhraseBased/Manager.cpp | 1 + moses2/PhraseBased/SentenceWithCandidates.cpp | 25 ++++++++-- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/cgmanifest.json b/cgmanifest.json index 6e7a9eafb..36cc60cff 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -1,33 +1,33 @@ -{ - "Registrations":[ - { - "component": { - "type": "git", - "git": { - "repositoryUrl": "https://github.com/moses-smt/mosesdecoder", - "commitHash": "78ca5f3cc5aa671a8a5d36c56452e217e6f00828" - } - } - }, - { - "component": { - "type": "git", - "git": { - "repositoryUrl": "https://git.code.sf.net/p/cmph/git", - "commitHash": "a250982ade093f4eed0552bbdd22dd7b0432007f" - } - } - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "xml-rpc-c", - "Version": "1.51.06", - "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz" - } - } - }, - ] -} - +{ + "Registrations":[ + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://github.com/moses-smt/mosesdecoder", + "commitHash": "78ca5f3cc5aa671a8a5d36c56452e217e6f00828" + } + } + }, + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://git.code.sf.net/p/cmph/git", + "commitHash": "a250982ade093f4eed0552bbdd22dd7b0432007f" + } + } + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "xml-rpc-c", + "Version": "1.51.06", + "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz" + } + } + }, + ] +} + diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index 670c33894..37c2ec669 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -59,6 +59,7 @@ void Manager::Init() InitPools(); FactorCollection &vocab = system.GetVocab(); + //TODO: need option to choose Sentence vs SentenceWithCandidates m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); m_bitmaps = new Bitmaps(GetPool()); diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index 4f55c5578..484d94cda 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -12,7 +12,7 @@ #include "../System.h" #include "../parameters/AllOptions.h" #include "../legacy/Util2.h" - +#include using namespace std; @@ -23,15 +23,30 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, const System &system, const std::string &str) { SentenceWithCandidates *ret; + + // unordered_map> ; + + // unordered_map s; + // s["abc"]=0.2; + // s["awc"]=0.4; + // s["abe"]=0.3; + // translation_candidates["src_1"]=s; + + // s.clear(); + // s["pqr"]=0.2; + // s["yen"]=0.4; + // s["dkg"]=0.5; + // translation_candidates["src_2"]=s; - vector result; - boost::split(result, str, boost::is_any_of("|||")); + vector input_parts; + boost::split(input_parts, str, boost::is_any_of("|||")); - if (result.size()!=2){ + if (input_parts.size()!=2){ exit(1); } - const string partstr = result[0] + const string partstr = input_parts[0] + parseCandidates(input_parts[1]) if (system.options.input.xml_policy) { // xml -- cgit v1.2.3 From fa11ef6e945731f2d2e881233cb6c052aed9040a Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Thu, 1 Oct 2020 16:48:19 +0530 Subject: basic parsing of input string --- moses2/PhraseBased/SentenceWithCandidates.cpp | 129 +++++++------------------- moses2/PhraseBased/SentenceWithCandidates.h | 8 +- 2 files changed, 40 insertions(+), 97 deletions(-) diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index f0da67a76..c4115cbe8 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -15,122 +15,63 @@ #include using namespace std; +using namespace boost; namespace Moses2 { +const string SentenceWithCandidates::INPUT_PART_DELIM = "@@@"; +const string SentenceWithCandidates::PT_LINE_DELIM = "$$$"; + SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str) { SentenceWithCandidates *ret; - // unordered_map> ; - - // unordered_map s; - // s["abc"]=0.2; - // s["awc"]=0.4; - // s["abe"]=0.3; - // translation_candidates["src_1"]=s; - - // s.clear(); - // s["pqr"]=0.2; - // s["yen"]=0.4; - // s["dkg"]=0.5; - // translation_candidates["src_2"]=s; - - vector input_parts; - boost::split(input_parts, str, boost::is_any_of("|||")); - - if (input_parts.size()!=2){ - exit(1); + // Break input into two parts: the parts are delimited by + typedef split_iterator string_split_iterator; + vector input_parts; + for(string_split_iterator It= make_split_iterator(str, first_finder(SentenceWithCandidates::INPUT_PART_DELIM, is_iequal())); + It!=string_split_iterator(); + ++It) + { + input_parts.push_back(copy_range(*It)); } - const string partstr = input_parts[0]; - // parseCandidates(input_parts[1]); - - if (system.options.input.xml_policy) { - // xml - ret = CreateFromStringXML(pool, vocab, system, partstr); - } else { - // no xml - //cerr << "PB SentenceWithCandidates" << endl; - std::vector toks = Tokenize(partstr); + cerr << "Number of subparts: " << input_parts.size() << endl; - size_t size = toks.size(); - ret = new (pool.Allocate()) SentenceWithCandidates(pool, size); - ret->PhraseImplTemplate::CreateFromString(vocab, system, toks, false); + if (input_parts.size() ==2 ) { + cerr << "correct number of parts" << endl ; + } else { + // TODO: how to handle wrong input format + cerr << "INCORRECT number of parts" << endl ; + exit(1); } - //cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl; - //cerr << "ret=" << ret->Debug(system) << endl; - - return ret; -} + trim(input_parts[0]); + trim(input_parts[1]); + cerr << "Input String: " << input_parts[0] << endl ; + cerr << "Phrase Table: " << input_parts[1] << endl ; -SentenceWithCandidates *SentenceWithCandidates::CreateFromStringXML(MemPool &pool, FactorCollection &vocab, - const System &system, const std::string &str) -{ - SentenceWithCandidates *ret; - - vector xmlOptions; - pugi::xml_document doc; - - string str2 = "" + str + ""; - pugi::xml_parse_result result = doc.load(str2.c_str(), - pugi::parse_cdata | pugi::parse_wconv_attribute | pugi::parse_eol | pugi::parse_comments); - pugi::xml_node topNode = doc.child("xml"); - - std::vector toks; - XMLParse(pool, system, 0, topNode, toks, xmlOptions); - - // debug - /* - cerr << "xmloptions:" << endl; - for (size_t i = 0; i < xmlOptions.size(); ++i) { - cerr << xmlOptions[i]->Debug(system) << endl; - } - */ + ///// Process the text part of the input + const string partstr = input_parts[0]; + + // no xml + //cerr << "PB SentenceWithCandidates" << endl; + std::vector toks = Tokenize(partstr); - // create words size_t size = toks.size(); ret = new (pool.Allocate()) SentenceWithCandidates(pool, size); ret->PhraseImplTemplate::CreateFromString(vocab, system, toks, false); - // xml - ret->Init(system, size, system.options.reordering.max_distortion); - - ReorderingConstraint &reorderingConstraint = ret->GetReorderingConstraint(); + //cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl; + //cerr << "ret=" << ret->Debug(system) << endl; - // set reordering walls, if "-monotone-at-punction" is set - if (system.options.reordering.monotone_at_punct && ret->GetSize()) { - reorderingConstraint.SetMonotoneAtPunctuation(*ret); - } - // set walls obtained from xml - for(size_t i=0; iGetNodeName(), "wall") == 0) { - if (xmlOption->startPos) { - UTIL_THROW_IF2(xmlOption->startPos > ret->GetSize(), "wall is beyond the SentenceWithCandidates"); // no buggy walls, please - reorderingConstraint.SetWall(xmlOption->startPos - 1, true); - } - } else if (strcmp(xmlOption->GetNodeName(), "zone") == 0) { - reorderingConstraint.SetZone( xmlOption->startPos, xmlOption->startPos + xmlOption->phraseSize -1 ); - } else if (strcmp(xmlOption->GetNodeName(), "ne") == 0) { - FactorType placeholderFactor = system.options.input.placeholder_factor; - UTIL_THROW_IF2(placeholderFactor == NOT_FOUND, - "Placeholder XML in input. Must have argument -placeholder-factor [NUM]"); - UTIL_THROW_IF2(xmlOption->phraseSize != 1, - "Placeholder must only cover 1 word"); - - const Factor *factor = vocab.AddFactor(xmlOption->GetEntity(), system, false); - (*ret)[xmlOption->startPos][placeholderFactor] = factor; - } else { - // default - forced translation. Add to class variable - ret->AddXMLOption(system, xmlOption); - } - } - reorderingConstraint.FinalizeWalls(); + //// Parse the phrase table of the input + ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n"); + cerr << "Extracted Phrase Table String" << endl; + cerr << ret->m_phraseTableString << endl; return ret; } diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 4dafca4af..114ff9e67 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -23,6 +23,9 @@ class SentenceWithCandidates: public Sentence { public: + static const std::string INPUT_PART_DELIM; + static const std::string PT_LINE_DELIM; + static SentenceWithCandidates *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str); @@ -33,9 +36,8 @@ public: virtual ~SentenceWithCandidates() {} -protected: - static SentenceWithCandidates *CreateFromStringXML(MemPool &pool, FactorCollection &vocab, - const System &system, const std::string &str); +private: + std::string m_phraseTableString; }; -- cgit v1.2.3 From e971f23eee8fc950ec56a3a4df0e68f281ceb8b7 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 1 Oct 2020 08:53:24 -0700 Subject: register MSPT --- moses2/FF/FeatureRegistry.cpp | 2 ++ moses2/Jamfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/moses2/FF/FeatureRegistry.cpp b/moses2/FF/FeatureRegistry.cpp index 3947d58c0..0ea56e6ce 100644 --- a/moses2/FF/FeatureRegistry.cpp +++ b/moses2/FF/FeatureRegistry.cpp @@ -4,6 +4,7 @@ #include "../TranslationModel/ProbingPT.h" #include "../TranslationModel/UnknownWordPenalty.h" #include "../TranslationModel/Transliteration.h" +#include "../TranslationModel/MSPT/MSPT.h" #include "../LM/KENLM.h" #include "../LM/KENLMBatch.h" @@ -56,6 +57,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(ProbingPT); MOSES_FNAME2("PhraseDictionaryTransliteration", Transliteration); MOSES_FNAME(UnknownWordPenalty); + MOSES_FNAME(MSPT); Add("KENLM", new KenFactory()); diff --git a/moses2/Jamfile b/moses2/Jamfile index e8a6457fd..cf20046c3 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -84,6 +84,8 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp TranslationModel/CompactPT/ThrowingFwrite.cpp + TranslationModel/MSPT/MSPT.cpp + parameters/AllOptions.cpp parameters/BookkeepingOptions.cpp parameters/ContextParameters.cpp -- cgit v1.2.3 From 1adc2f9f94ca433095ca6d75a57acb2235a0a03f Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 1 Oct 2020 10:03:32 -0700 Subject: InitializeForInput() --- moses2/FF/FeatureFunction.h | 3 +++ moses2/FF/FeatureFunctions.cpp | 7 +++++++ moses2/FF/FeatureFunctions.h | 6 ++++-- moses2/ManagerBase.cpp | 1 + moses2/ManagerBase.h | 2 +- moses2/System.cpp | 2 +- moses2/TranslationModel/MSPT/MSPT.cpp | 5 +++++ moses2/TranslationModel/MSPT/MSPT.h | 2 ++ 8 files changed, 24 insertions(+), 4 deletions(-) diff --git a/moses2/FF/FeatureFunction.h b/moses2/FF/FeatureFunction.h index 102bda8f1..fa0f1fbe9 100644 --- a/moses2/FF/FeatureFunction.h +++ b/moses2/FF/FeatureFunction.h @@ -25,6 +25,7 @@ class TargetPhraseImpl; class Scores; class ManagerBase; class MemPool; +class InputType; namespace SCFG { @@ -95,6 +96,8 @@ public: const SCFG::TargetPhrases &tps, const Phrase &sourcePhrase) const { } + virtual void InitializeForInput(const InputType &input) { }; + // clean up temporary memory, called after processing each sentence virtual void CleanUpAfterSentenceProcessing() const { } diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp index 0e61fb0e4..12cdd8d24 100644 --- a/moses2/FF/FeatureFunctions.cpp +++ b/moses2/FF/FeatureFunctions.cpp @@ -229,6 +229,13 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const } } +void FeatureFunctions::InitializeForInput(const InputType &input) +{ + BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) { + ff->InitializeForInput(input); + } +} + void FeatureFunctions::CleanUpAfterSentenceProcessing() const { BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) { diff --git a/moses2/FF/FeatureFunctions.h b/moses2/FF/FeatureFunctions.h index 6a3f9bb78..6714d57cc 100644 --- a/moses2/FF/FeatureFunctions.h +++ b/moses2/FF/FeatureFunctions.h @@ -31,6 +31,7 @@ class Scores; class Hypothesis; class UnknownWordPenalty; class Weights; +class InputType; namespace SCFG { @@ -47,7 +48,7 @@ public: FeatureFunctions(System &system); virtual ~FeatureFunctions(); - const std::vector &GetFeatureFunctions() const { + const std::vector &GetFeatureFunctions() const { return m_featureFunctions; } @@ -86,12 +87,13 @@ public: void EvaluateWhenAppliedBatch(const Batch &batch) const; + void InitializeForInput(const InputType &input); void CleanUpAfterSentenceProcessing() const; void ShowWeights(const Weights &allWeights); protected: - std::vector m_featureFunctions; + std::vector m_featureFunctions; std::vector m_statefulFeatureFunctions; std::vector m_withPhraseTableInd; const UnknownWordPenalty *m_unkWP; diff --git a/moses2/ManagerBase.cpp b/moses2/ManagerBase.cpp index f40aa7b2f..06365df54 100644 --- a/moses2/ManagerBase.cpp +++ b/moses2/ManagerBase.cpp @@ -30,6 +30,7 @@ ManagerBase::ManagerBase(System &sys, const TranslationTask &task, ,m_systemPool(NULL) ,m_hypoRecycle(NULL) { + system.featureFunctions.InitializeForInput(*m_input); } ManagerBase::~ManagerBase() diff --git a/moses2/ManagerBase.h b/moses2/ManagerBase.h index cb8ee019c..d0740109c 100644 --- a/moses2/ManagerBase.h +++ b/moses2/ManagerBase.h @@ -33,7 +33,7 @@ class HypothesisBase; class ManagerBase { public: - const System &system; + System &system; const TranslationTask &task; mutable ArcLists arcLists; diff --git a/moses2/System.cpp b/moses2/System.cpp index fefb61ba4..91c247b75 100644 --- a/moses2/System.cpp +++ b/moses2/System.cpp @@ -97,7 +97,7 @@ void System::LoadWeights() const WeightMap &allWeights = params.GetAllWeights(); // check all weights are there for all FF - const std::vector &ffs = featureFunctions.GetFeatureFunctions(); + const std::vector &ffs = featureFunctions.GetFeatureFunctions(); BOOST_FOREACH(const FeatureFunction *ff, ffs) { if (ff->IsTuneable()) { const std::string &ffName = ff->GetName(); diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index c905d5240..04a900812 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -150,6 +150,11 @@ void MSPT::Load(System &system) */ } +void MSPT::InitializeForInput(const InputType &input) +{ + cerr << "InitializeForInput" << endl; +} + TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, InputPath &inputPath) const { diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index d3946d353..a7fd6e2da 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -66,6 +66,8 @@ public: const SCFG::Stacks &stacks, SCFG::InputPath &path) const; + virtual void InitializeForInput(const InputType &input); + protected: PBNODE *m_rootPb; SCFGNODE *m_rootSCFG; -- cgit v1.2.3 From 95d93f8bd1be450c7713caf9584a37b9e211231f Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 1 Oct 2020 10:10:20 -0700 Subject: make sure InitializeForInput is called --- moses2/TranslationModel/MSPT/MSPT.cpp | 100 ---------------------------------- moses2/TranslationModel/MSPT/MSPT.h | 1 - 2 files changed, 101 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 04a900812..e7ce62a62 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -50,106 +50,6 @@ MSPT::~MSPT() delete m_rootSCFG; } -void MSPT::Load(System &system) -{ - FactorCollection &vocab = system.GetVocab(); - MemPool &systemPool = system.GetSystemPool(); - MemPool tmpSourcePool; - - if (system.isPb) { - m_rootPb = new PBNODE(); - } else { - m_rootSCFG = new SCFGNODE(); - //cerr << "m_rootSCFG=" << m_rootSCFG << endl; - } - - vector toks; - size_t lineNum = 0; - InputFileStream strme(m_path); - string line; - while (getline(strme, line)) { - if (++lineNum % 1000000 == 0) { - cerr << lineNum << " "; - } - toks.clear(); - TokenizeMultiCharSeparator(toks, line, "|||"); - UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); - //cerr << "line=" << line << endl; - //cerr << "system.isPb=" << system.isPb << endl; - - if (system.isPb) { - PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); - //cerr << "created soure" << endl; - TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, - toks[1]); - //cerr << "created target" << endl; - target->GetScores().CreateFromString(toks[2], *this, system, true); - //cerr << "created scores:" << *target << endl; - - if (toks.size() >= 4) { - //cerr << "alignstr=" << toks[3] << endl; - target->SetAlignmentInfo(toks[3]); - } - - // properties - if (toks.size() == 7) { - //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); - //strcpy(target->properties, toks[6].c_str()); - } - - system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, - *target); - //cerr << "EvaluateInIsolation:" << *target << endl; - m_rootPb->AddRule(m_input, *source, target); - - //cerr << "target=" << target->Debug(system) << endl; - } else { - SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); - //cerr << "created source:" << *source << endl; - SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, - system, toks[1]); - - //cerr << "created target " << *target << " source=" << *source << endl; - - target->GetScores().CreateFromString(toks[2], *this, system, true); - //cerr << "created scores:" << *target << endl; - - //vector scores = Tokenize(toks[2]); - //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; - - target->SetAlignmentInfo(toks[3]); - - // properties - if (toks.size() == 7) { - //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); - //strcpy(target->properties, toks[6].c_str()); - } - - system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, - *target); - //cerr << "EvaluateInIsolation:" << *target << endl; - m_rootSCFG->AddRule(m_input, *source, target); - } - } - - if (system.isPb) { - m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); - //cerr << "root=" << &m_rootPb << endl; - } else { - m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); - //cerr << "root=" << &m_rootPb << endl; - } - /* - BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { - const Word &word = valPair.first; - cerr << word << " "; - } - cerr << endl; - */ -} - void MSPT::InitializeForInput(const InputType &input) { cerr << "InitializeForInput" << endl; diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index a7fd6e2da..165565791 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -51,7 +51,6 @@ public: MSPT(size_t startInd, const std::string &line); virtual ~MSPT(); - virtual void Load(System &system); virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, InputPath &inputPath) const; -- cgit v1.2.3 From 2e94ba7a9f7f66042fb31704645997c18e9cd244 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 1 Oct 2020 10:33:59 -0700 Subject: add input arg to CleanUpAfterSentenceProcessing --- moses2/FF/FeatureFunction.h | 2 +- moses2/FF/FeatureFunctions.cpp | 4 ++-- moses2/FF/FeatureFunctions.h | 2 +- moses2/ManagerBase.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/moses2/FF/FeatureFunction.h b/moses2/FF/FeatureFunction.h index fa0f1fbe9..34742efb1 100644 --- a/moses2/FF/FeatureFunction.h +++ b/moses2/FF/FeatureFunction.h @@ -99,7 +99,7 @@ public: virtual void InitializeForInput(const InputType &input) { }; // clean up temporary memory, called after processing each sentence - virtual void CleanUpAfterSentenceProcessing() const { + virtual void CleanUpAfterSentenceProcessing(const InputType &input) const { } protected: diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp index 12cdd8d24..efe018b7b 100644 --- a/moses2/FF/FeatureFunctions.cpp +++ b/moses2/FF/FeatureFunctions.cpp @@ -236,10 +236,10 @@ void FeatureFunctions::InitializeForInput(const InputType &input) } } -void FeatureFunctions::CleanUpAfterSentenceProcessing() const +void FeatureFunctions::CleanUpAfterSentenceProcessing(const InputType &input) const { BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) { - ff->CleanUpAfterSentenceProcessing(); + ff->CleanUpAfterSentenceProcessing(input); } } diff --git a/moses2/FF/FeatureFunctions.h b/moses2/FF/FeatureFunctions.h index 6714d57cc..4c15d674a 100644 --- a/moses2/FF/FeatureFunctions.h +++ b/moses2/FF/FeatureFunctions.h @@ -88,7 +88,7 @@ public: void EvaluateWhenAppliedBatch(const Batch &batch) const; void InitializeForInput(const InputType &input); - void CleanUpAfterSentenceProcessing() const; + void CleanUpAfterSentenceProcessing(const InputType &input) const; void ShowWeights(const Weights &allWeights); diff --git a/moses2/ManagerBase.cpp b/moses2/ManagerBase.cpp index 06365df54..0ab60f9f1 100644 --- a/moses2/ManagerBase.cpp +++ b/moses2/ManagerBase.cpp @@ -35,7 +35,7 @@ ManagerBase::ManagerBase(System &sys, const TranslationTask &task, ManagerBase::~ManagerBase() { - system.featureFunctions.CleanUpAfterSentenceProcessing(); + system.featureFunctions.CleanUpAfterSentenceProcessing(*m_input); if (m_pool) { GetPool().Reset(); -- cgit v1.2.3 From a269d24db253ea54a8c67c4d36d68d1ff7932c8b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 1 Oct 2020 10:39:47 -0700 Subject: InitializeForInput() --- moses2/TranslationModel/MSPT/MSPT.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index e7ce62a62..b1f9b6595 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -52,7 +52,7 @@ MSPT::~MSPT() void MSPT::InitializeForInput(const InputType &input) { - cerr << "InitializeForInput" << endl; + cerr << "InitializeForInput MSPT" << endl; } TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, -- cgit v1.2.3 From 2cb12decc8abcb60c84f92a9d82271782bfd6fe4 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Sun, 4 Oct 2020 22:43:48 +0530 Subject: dynamic cast to SentenceWithCandidates (failed) --- moses2/PhraseBased/SentenceWithCandidates.cpp | 5 +- moses2/PhraseBased/SentenceWithCandidates.h | 4 + moses2/TranslationModel/MSPT/MSPT.cpp | 108 ++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index c4115cbe8..cb96a9d92 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -70,8 +70,9 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, //// Parse the phrase table of the input ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n"); - cerr << "Extracted Phrase Table String" << endl; - cerr << ret->m_phraseTableString << endl; + // ret->m_phraseTableString="constant phrase table"; +// cerr << "Extracted Phrase Table String: " << ret->m_phraseTableString << endl; + cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl; return ret; } diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 114ff9e67..5cc34590d 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -36,6 +36,10 @@ public: virtual ~SentenceWithCandidates() {} + std::string virtual getPhraseTableString() const{ + return m_phraseTableString; + } + private: std::string m_phraseTableString; diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index e7ce62a62..8bdbf46c0 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -19,6 +19,7 @@ #include "../../PhraseBased/InputPath.h" #include "../../PhraseBased/TargetPhraseImpl.h" #include "../../PhraseBased/TargetPhrases.h" +#include "../../PhraseBased/SentenceWithCandidates.h" #include "../../SCFG/PhraseImpl.h" #include "../../SCFG/TargetPhraseImpl.h" @@ -50,9 +51,116 @@ MSPT::~MSPT() delete m_rootSCFG; } +// void MSPT::CreatePTForInput(string phraseTableString) +// { +// FactorCollection &vocab = system.GetVocab(); +// MemPool &systemPool = system.GetSystemPool(); +// MemPool tmpSourcePool; + +// if (system.isPb) { +// m_rootPb = new PBNODE(); +// } else { +// m_rootSCFG = new SCFGNODE(); +// //cerr << "m_rootSCFG=" << m_rootSCFG << endl; +// } + +// vector toks; +// size_t lineNum = 0; +// InputFileStream strme(m_path); +// string line; +// while (getline(strme, line)) { +// if (++lineNum % 1000000 == 0) { +// cerr << lineNum << " "; +// } +// toks.clear(); +// TokenizeMultiCharSeparator(toks, line, "|||"); +// UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); +// //cerr << "line=" << line << endl; +// //cerr << "system.isPb=" << system.isPb << endl; + +// if (system.isPb) { +// PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, +// toks[0]); +// //cerr << "created soure" << endl; +// TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, +// toks[1]); +// //cerr << "created target" << endl; +// target->GetScores().CreateFromString(toks[2], *this, system, true); +// //cerr << "created scores:" << *target << endl; + +// if (toks.size() >= 4) { +// //cerr << "alignstr=" << toks[3] << endl; +// target->SetAlignmentInfo(toks[3]); +// } + +// // properties +// if (toks.size() == 7) { +// //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); +// //strcpy(target->properties, toks[6].c_str()); +// } + +// system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, +// *target); +// //cerr << "EvaluateInIsolation:" << *target << endl; +// m_rootPb->AddRule(m_input, *source, target); + +// //cerr << "target=" << target->Debug(system) << endl; +// } else { +// SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, +// toks[0]); +// //cerr << "created source:" << *source << endl; +// SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, +// system, toks[1]); + +// //cerr << "created target " << *target << " source=" << *source << endl; + +// target->GetScores().CreateFromString(toks[2], *this, system, true); +// //cerr << "created scores:" << *target << endl; + +// //vector scores = Tokenize(toks[2]); +// //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; + +// target->SetAlignmentInfo(toks[3]); + +// // properties +// if (toks.size() == 7) { +// //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); +// //strcpy(target->properties, toks[6].c_str()); +// } + +// system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, +// *target); +// //cerr << "EvaluateInIsolation:" << *target << endl; +// m_rootSCFG->AddRule(m_input, *source, target); +// } +// } + +// if (system.isPb) { +// m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); +// //cerr << "root=" << &m_rootPb << endl; +// } else { +// m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); +// //cerr << "root=" << &m_rootPb << endl; +// } +// /* +// BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { +// const Word &word = valPair.first; +// cerr << word << " "; +// } +// cerr << endl; +// */ + +// } + void MSPT::InitializeForInput(const InputType &input) { cerr << "InitializeForInput" << endl; + + // downcast to SentenceWithCandidates + const SentenceWithCandidates& inputObj = dynamic_cast(input); + cerr << "Casting done." << endl; + // cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; + } TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, -- cgit v1.2.3 From 083b6bf560afb959640ed7c37562e964cf0ae1fd Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 4 Oct 2020 11:39:56 -0700 Subject: add system --- moses2/FF/FeatureFunction.h | 4 ++-- moses2/FF/FeatureFunctions.cpp | 4 ++-- moses2/TranslationModel/MSPT/MSPT.cpp | 11 +++++++---- moses2/TranslationModel/MSPT/MSPT.h | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/moses2/FF/FeatureFunction.h b/moses2/FF/FeatureFunction.h index 34742efb1..828a712b9 100644 --- a/moses2/FF/FeatureFunction.h +++ b/moses2/FF/FeatureFunction.h @@ -96,10 +96,10 @@ public: const SCFG::TargetPhrases &tps, const Phrase &sourcePhrase) const { } - virtual void InitializeForInput(const InputType &input) { }; + virtual void InitializeForInput(const System &system, const InputType &input) { }; // clean up temporary memory, called after processing each sentence - virtual void CleanUpAfterSentenceProcessing(const InputType &input) const { + virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { } protected: diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp index efe018b7b..39e2436b6 100644 --- a/moses2/FF/FeatureFunctions.cpp +++ b/moses2/FF/FeatureFunctions.cpp @@ -232,14 +232,14 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const void FeatureFunctions::InitializeForInput(const InputType &input) { BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) { - ff->InitializeForInput(input); + ff->InitializeForInput(m_system, input); } } void FeatureFunctions::CleanUpAfterSentenceProcessing(const InputType &input) const { BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) { - ff->CleanUpAfterSentenceProcessing(input); + ff->CleanUpAfterSentenceProcessing(m_system, input); } } diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 17dea92a0..8d33071f6 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -28,6 +28,7 @@ #include "../../SCFG/Stacks.h" #include "../../SCFG/Manager.h" +#include "../../PhraseBased/SentenceWithCandidates.h" using namespace std; @@ -152,14 +153,16 @@ MSPT::~MSPT() // } -void MSPT::InitializeForInput(const InputType &input) +void MSPT::InitializeForInput(const System &system, const InputType &input) { cerr << "InitializeForInput MSPT" << endl; + cerr << &input << endl; // downcast to SentenceWithCandidates - const SentenceWithCandidates& inputObj = dynamic_cast(input); - cerr << "Casting done." << endl; - // cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; + //const SentenceWithCandidates *inputObj = static_cast(&input); + const SentenceWithCandidates* inputObj = dynamic_cast(&input); + cerr << "Casting done." << endl << flush; + cerr << "PhraseTableString member: " << inputObj->getPhraseTableString() << endl; } diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index 165565791..b3ff99c91 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -65,7 +65,7 @@ public: const SCFG::Stacks &stacks, SCFG::InputPath &path) const; - virtual void InitializeForInput(const InputType &input); + virtual void InitializeForInput(const System &system, const InputType &input); protected: PBNODE *m_rootPb; -- cgit v1.2.3 From 0216957a371e964a53aaa17372a727bfe8eeb6e7 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 4 Oct 2020 11:58:31 -0700 Subject: debug --- moses2/InputType.cpp | 8 ++++++++ moses2/InputType.h | 2 ++ moses2/ManagerBase.cpp | 2 ++ moses2/PhraseBased/Manager.cpp | 1 + moses2/PhraseBased/SentenceWithCandidates.cpp | 5 +++++ moses2/PhraseBased/SentenceWithCandidates.h | 1 + moses2/TranslationModel/MSPT/MSPT.cpp | 5 +++-- 7 files changed, 22 insertions(+), 2 deletions(-) diff --git a/moses2/InputType.cpp b/moses2/InputType.cpp index 60664a85b..af5b61ff9 100644 --- a/moses2/InputType.cpp +++ b/moses2/InputType.cpp @@ -7,6 +7,9 @@ #include "InputType.h" #include "System.h" +#include + +using namespace std; namespace Moses2 { @@ -89,4 +92,9 @@ bool InputType::XmlOverlap(size_t startPos, size_t endPos) const return false; } +std::string InputType::Debug(const System &system) const +{ + cerr << "InputType::Debug" << endl; +} + } /* namespace Moses2 */ diff --git a/moses2/InputType.h b/moses2/InputType.h index 8813bc484..b4f901ac6 100644 --- a/moses2/InputType.h +++ b/moses2/InputType.h @@ -73,6 +73,8 @@ public: //! Returns true if there were any XML tags parsed that at least partially covered the range passed bool XmlOverlap(size_t startPos, size_t endPos) const; + virtual std::string Debug(const System &system) const; + protected: ReorderingConstraint m_reorderingConstraint; /**< limits on reordering specified either by "-mp" switch or xml tags */ Vector m_xmlOptions; diff --git a/moses2/ManagerBase.cpp b/moses2/ManagerBase.cpp index 0ab60f9f1..705c766d8 100644 --- a/moses2/ManagerBase.cpp +++ b/moses2/ManagerBase.cpp @@ -29,7 +29,9 @@ ManagerBase::ManagerBase(System &sys, const TranslationTask &task, ,m_pool(NULL) ,m_systemPool(NULL) ,m_hypoRecycle(NULL) + ,m_input(NULL) { + //cerr << "ManagerBase::ManagerBase " << m_input->Debug(sys) << endl << flush; system.featureFunctions.InitializeForInput(*m_input); } diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index bb3c130c5..a317385d9 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -62,6 +62,7 @@ void Manager::Init() FactorCollection &vocab = system.GetVocab(); //TODO: need option to choose Sentence vs SentenceWithCandidates m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); + cerr << "Manager::Init: " << m_input->Debug(sys) << endl << flush; m_bitmaps = new Bitmaps(GetPool()); diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index cb96a9d92..a0ff5a82b 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -77,5 +77,10 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, return ret; } +std::string SentenceWithCandidates::Debug(const System &system) const +{ + cerr << "SentenceWithCandidates::Debug" << endl; +} + } /* namespace Moses2 */ diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 5cc34590d..038fb5f1e 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -36,6 +36,7 @@ public: virtual ~SentenceWithCandidates() {} + virtual std::string Debug(const System &system) const; std::string virtual getPhraseTableString() const{ return m_phraseTableString; } diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 8d33071f6..e94078a7b 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -156,8 +156,9 @@ MSPT::~MSPT() void MSPT::InitializeForInput(const System &system, const InputType &input) { cerr << "InitializeForInput MSPT" << endl; - cerr << &input << endl; - + cerr << input.Debug(system) << endl << flush; + cerr << "HH1" << endl; + // downcast to SentenceWithCandidates //const SentenceWithCandidates *inputObj = static_cast(&input); const SentenceWithCandidates* inputObj = dynamic_cast(&input); -- cgit v1.2.3 From 88e7ab5ec427b165ff4ac38934d227e24aae413d Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 4 Oct 2020 12:04:14 -0700 Subject: move InitializeForInput to Init --- moses2/ManagerBase.cpp | 2 -- moses2/PhraseBased/Manager.cpp | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moses2/ManagerBase.cpp b/moses2/ManagerBase.cpp index 705c766d8..41d3a0394 100644 --- a/moses2/ManagerBase.cpp +++ b/moses2/ManagerBase.cpp @@ -31,8 +31,6 @@ ManagerBase::ManagerBase(System &sys, const TranslationTask &task, ,m_hypoRecycle(NULL) ,m_input(NULL) { - //cerr << "ManagerBase::ManagerBase " << m_input->Debug(sys) << endl << flush; - system.featureFunctions.InitializeForInput(*m_input); } ManagerBase::~ManagerBase() diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index a317385d9..3d6391b2b 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -62,7 +62,8 @@ void Manager::Init() FactorCollection &vocab = system.GetVocab(); //TODO: need option to choose Sentence vs SentenceWithCandidates m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); - cerr << "Manager::Init: " << m_input->Debug(sys) << endl << flush; + cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush; + system.featureFunctions.InitializeForInput(*m_input); m_bitmaps = new Bitmaps(GetPool()); -- cgit v1.2.3 From 5e38a00a5fc27291f45adecfadf006818c892306 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 4 Oct 2020 12:17:44 -0700 Subject: debug --- moses2/PhraseBased/Manager.cpp | 2 +- moses2/PhraseBased/SentenceWithCandidates.cpp | 11 +++++++++++ moses2/PhraseBased/SentenceWithCandidates.h | 8 ++------ moses2/TranslationModel/MSPT/MSPT.cpp | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index 3d6391b2b..cce30efb1 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -62,7 +62,7 @@ void Manager::Init() FactorCollection &vocab = system.GetVocab(); //TODO: need option to choose Sentence vs SentenceWithCandidates m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); - cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush; + //cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush; system.featureFunctions.InitializeForInput(*m_input); m_bitmaps = new Bitmaps(GetPool()); diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index a0ff5a82b..6e4190a4e 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -77,6 +77,17 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, return ret; } +SentenceWithCandidates::SentenceWithCandidates(MemPool &pool, size_t size) +:Sentence(pool, size) +{ + cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl; +} + +SentenceWithCandidates::~SentenceWithCandidates() +{ + cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl; +} + std::string SentenceWithCandidates::Debug(const System &system) const { cerr << "SentenceWithCandidates::Debug" << endl; diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h index 038fb5f1e..fb550d577 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.h +++ b/moses2/PhraseBased/SentenceWithCandidates.h @@ -29,12 +29,8 @@ public: static SentenceWithCandidates *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str); - SentenceWithCandidates(MemPool &pool, size_t size) - :Sentence(pool, size) - {} - - virtual ~SentenceWithCandidates() - {} + SentenceWithCandidates(MemPool &pool, size_t size); + virtual ~SentenceWithCandidates(); virtual std::string Debug(const System &system) const; std::string virtual getPhraseTableString() const{ diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index e94078a7b..cd1f16e75 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -157,13 +157,13 @@ void MSPT::InitializeForInput(const System &system, const InputType &input) { cerr << "InitializeForInput MSPT" << endl; cerr << input.Debug(system) << endl << flush; - cerr << "HH1" << endl; + cerr << "HH1" << endl << flush; // downcast to SentenceWithCandidates - //const SentenceWithCandidates *inputObj = static_cast(&input); - const SentenceWithCandidates* inputObj = dynamic_cast(&input); + //const SentenceWithCandidates &inputObj = static_cast(input); + const SentenceWithCandidates &inputObj = dynamic_cast(input); cerr << "Casting done." << endl << flush; - cerr << "PhraseTableString member: " << inputObj->getPhraseTableString() << endl; + cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; } -- cgit v1.2.3 From 6801d65c58c8d47e7f9424fe6c842fc046b84d62 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Mon, 5 Oct 2020 08:25:29 +0530 Subject: some code for parsing input --- moses2/TranslationModel/MSPT/MSPT.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 17dea92a0..529efbb69 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include "MSPT.h" #include "../../PhraseBased/PhraseImpl.h" @@ -66,7 +67,7 @@ MSPT::~MSPT() // vector toks; // size_t lineNum = 0; -// InputFileStream strme(m_path); +// istringstream strme(phraseTableString); // string line; // while (getline(strme, line)) { // if (++lineNum % 1000000 == 0) { @@ -159,7 +160,7 @@ void MSPT::InitializeForInput(const InputType &input) // downcast to SentenceWithCandidates const SentenceWithCandidates& inputObj = dynamic_cast(input); cerr << "Casting done." << endl; - // cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; + cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; } -- cgit v1.2.3 From 6e78aae34fc58b7e98d463ac56d8485f66e0ec77 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 5 Oct 2020 09:08:16 -0700 Subject: abort SCFG --- moses2/TranslationModel/MSPT/MSPT.cpp | 80 ++--------------------------------- 1 file changed, 3 insertions(+), 77 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 90feb3489..ab16c3255 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -181,10 +181,7 @@ void MSPT::InitActiveChart( const SCFG::Manager &mgr, SCFG::InputPath &path) const { - size_t ptInd = GetPtInd(); - ActiveChartEntryMem *chartEntry = new (pool.Allocate()) ActiveChartEntryMem(pool, *m_rootSCFG); - path.AddActiveChartEntry(ptInd, chartEntry); - //cerr << "InitActiveChart=" << path << endl; + abort(); } void MSPT::Lookup(MemPool &pool, @@ -193,38 +190,7 @@ void MSPT::Lookup(MemPool &pool, const SCFG::Stacks &stacks, SCFG::InputPath &path) const { - if (path.range.GetNumWordsCovered() > maxChartSpan) { - return; - } - - size_t endPos = path.range.GetEndPos(); - - const SCFG::InputPath *prevPath = static_cast(path.prefixPath); - UTIL_THROW_IF2(prevPath == NULL, "prefixPath == NULL"); - - // TERMINAL - const SCFG::Word &lastWord = path.subPhrase.Back(); - - const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1); - - //cerr << "BEFORE LookupGivenWord=" << *prevPath << endl; - LookupGivenWord(pool, mgr, *prevPath, lastWord, NULL, subPhrasePath.range, path); - //cerr << "AFTER LookupGivenWord=" << *prevPath << endl; - - // NON-TERMINAL - //const SCFG::InputPath *prefixPath = static_cast(path.prefixPath); - while (prevPath) { - const Range &prevRange = prevPath->range; - //cerr << "prevRange=" << prevRange << endl; - - size_t startPos = prevRange.GetEndPos() + 1; - size_t ntSize = endPos - startPos + 1; - const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(startPos, ntSize); - - LookupNT(pool, mgr, subPhrasePath.range, *prevPath, stacks, path); - - prevPath = static_cast(prevPath->prefixPath); - } + abort(); } void MSPT::LookupGivenNode( @@ -236,47 +202,7 @@ void MSPT::LookupGivenNode( const Moses2::Range &subPhraseRange, SCFG::InputPath &outPath) const { - const ActiveChartEntryMem &prevEntryCast = static_cast(prevEntry); - - const SCFGNODE &prevNode = prevEntryCast.node; - UTIL_THROW_IF2(&prevNode == NULL, "node == NULL"); - - size_t ptInd = GetPtInd(); - const SCFGNODE *nextNode = prevNode.Find(m_input, wordSought); - - /* - if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) { - cerr << "range=" << outPath.range - << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) - << " wordSought=" << wordSought.Debug(mgr.system) - << " nextNode=" << nextNode - << endl; - } - */ - if (nextNode) { - // new entries - ActiveChartEntryMem *chartEntry = new (pool.Allocate()) ActiveChartEntryMem(pool, *nextNode, prevEntry); - - chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); - //cerr << "AFTER Add=" << symbolBind << endl; - - outPath.AddActiveChartEntry(ptInd, chartEntry); - - const SCFG::TargetPhrases *tps = nextNode->GetTargetPhrases(); - if (tps) { - // there are some rules - /* - cerr << "outPath=" << outPath.range - << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system) - << " pt=" << GetPtInd() - << " tps=" << tps->Debug(mgr.system) << endl; - */ - outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); - - } - - //cerr << "AFTER outPath=" << outPath << endl; - } + abort(); } } -- cgit v1.2.3 From 0b0d5e84926983330d57120987a480e96867b943 Mon Sep 17 00:00:00 2001 From: Anoop Kunchukuttan Date: Mon, 5 Oct 2020 22:38:15 +0530 Subject: MSPT changes for parsing phrase table string --- moses2/TranslationModel/MSPT/MSNode.h | 2 +- moses2/TranslationModel/MSPT/MSPT.cpp | 206 +++++++++++++++++----------------- moses2/TranslationModel/MSPT/MSPT.h | 2 + 3 files changed, 109 insertions(+), 101 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSNode.h b/moses2/TranslationModel/MSPT/MSNode.h index ad6d0842d..b02422aa5 100644 --- a/moses2/TranslationModel/MSPT/MSNode.h +++ b/moses2/TranslationModel/MSPT/MSNode.h @@ -67,7 +67,7 @@ public: return m_targetPhrases; } - void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) { + void SortAndPrune(size_t tableLimit, MemPool &pool, const System &system) { BOOST_FOREACH(typename Children::value_type &val, m_children) { Node &child = val.second; child.SortAndPrune(tableLimit, pool, system); diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 90feb3489..665d6dbea 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -53,106 +53,108 @@ MSPT::~MSPT() delete m_rootSCFG; } -// void MSPT::CreatePTForInput(string phraseTableString) -// { -// FactorCollection &vocab = system.GetVocab(); -// MemPool &systemPool = system.GetSystemPool(); -// MemPool tmpSourcePool; - -// if (system.isPb) { -// m_rootPb = new PBNODE(); -// } else { -// m_rootSCFG = new SCFGNODE(); -// //cerr << "m_rootSCFG=" << m_rootSCFG << endl; -// } - -// vector toks; -// size_t lineNum = 0; -// istringstream strme(phraseTableString); -// string line; -// while (getline(strme, line)) { -// if (++lineNum % 1000000 == 0) { -// cerr << lineNum << " "; -// } -// toks.clear(); -// TokenizeMultiCharSeparator(toks, line, "|||"); -// UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); -// //cerr << "line=" << line << endl; -// //cerr << "system.isPb=" << system.isPb << endl; - -// if (system.isPb) { -// PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, -// toks[0]); -// //cerr << "created soure" << endl; -// TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, -// toks[1]); -// //cerr << "created target" << endl; -// target->GetScores().CreateFromString(toks[2], *this, system, true); -// //cerr << "created scores:" << *target << endl; - -// if (toks.size() >= 4) { -// //cerr << "alignstr=" << toks[3] << endl; -// target->SetAlignmentInfo(toks[3]); -// } - -// // properties -// if (toks.size() == 7) { -// //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); -// //strcpy(target->properties, toks[6].c_str()); -// } - -// system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, -// *target); -// //cerr << "EvaluateInIsolation:" << *target << endl; -// m_rootPb->AddRule(m_input, *source, target); - -// //cerr << "target=" << target->Debug(system) << endl; -// } else { -// SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, -// toks[0]); -// //cerr << "created source:" << *source << endl; -// SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, -// system, toks[1]); - -// //cerr << "created target " << *target << " source=" << *source << endl; - -// target->GetScores().CreateFromString(toks[2], *this, system, true); -// //cerr << "created scores:" << *target << endl; - -// //vector scores = Tokenize(toks[2]); -// //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; - -// target->SetAlignmentInfo(toks[3]); - -// // properties -// if (toks.size() == 7) { -// //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); -// //strcpy(target->properties, toks[6].c_str()); -// } - -// system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, -// *target); -// //cerr << "EvaluateInIsolation:" << *target << endl; -// m_rootSCFG->AddRule(m_input, *source, target); -// } -// } - -// if (system.isPb) { -// m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); -// //cerr << "root=" << &m_rootPb << endl; -// } else { -// m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); -// //cerr << "root=" << &m_rootPb << endl; -// } -// /* -// BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { -// const Word &word = valPair.first; -// cerr << word << " "; -// } -// cerr << endl; -// */ - -// } +void MSPT::CreatePTForInput(const System &system, string phraseTableString) +{ + cerr << "In CreatePTForInput" << endl << flush; + + FactorCollection &vocab = system.GetVocab(); + MemPool &systemPool = system.GetSystemPool(); + MemPool tmpSourcePool; + + if (system.isPb) { + m_rootPb = new PBNODE(); + } else { + m_rootSCFG = new SCFGNODE(); + //cerr << "m_rootSCFG=" << m_rootSCFG << endl; + } + + vector toks; + size_t lineNum = 0; + istringstream strme(phraseTableString); + string line; + while (getline(strme, line)) { + if (++lineNum % 1000000 == 0) { + cerr << lineNum << " "; + } + toks.clear(); + TokenizeMultiCharSeparator(toks, line, "|||"); + UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); + //cerr << "line=" << line << endl; + //cerr << "system.isPb=" << system.isPb << endl; + + if (system.isPb) { + PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created soure" << endl; + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, + toks[1]); + //cerr << "created target" << endl; + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + if (toks.size() >= 4) { + //cerr << "alignstr=" << toks[3] << endl; + target->SetAlignmentInfo(toks[3]); + } + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootPb->AddRule(m_input, *source, target); + + //cerr << "target=" << target->Debug(system) << endl; + } else { + SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created source:" << *source << endl; + SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, + system, toks[1]); + + //cerr << "created target " << *target << " source=" << *source << endl; + + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + //vector scores = Tokenize(toks[2]); + //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; + + target->SetAlignmentInfo(toks[3]); + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootSCFG->AddRule(m_input, *source, target); + } + } + + if (system.isPb) { + m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } else { + m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } + /* + BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { + const Word &word = valPair.first; + cerr << word << " "; + } + cerr << endl; + */ + +} void MSPT::InitializeForInput(const System &system, const InputType &input) { @@ -166,6 +168,10 @@ void MSPT::InitializeForInput(const System &system, const InputType &input) cerr << "Casting done." << endl << flush; cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; + cerr << "Hardcoding sample PhraseTableString" << endl << flush; + string phraseTableString="a ||| x ||| 0.4 $$$ a ||| y ||| 0.6 $$$ b ||| y ||| 0.1 $$$ b ||| z ||| 0.9"; + CreatePTForInput(system,phraseTableString); + } TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index b3ff99c91..744158ea0 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -80,6 +80,8 @@ protected: const Moses2::Range &subPhraseRange, SCFG::InputPath &outPath) const; + void CreatePTForInput(const System &system, std::string phraseTableString); + }; } -- cgit v1.2.3 From 83dd5fb7ff999199efdeada5e93e67ad12bff548 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 5 Oct 2020 12:47:13 -0700 Subject: MSPT works --- moses2/PhraseBased/SentenceWithCandidates.cpp | 16 +++++++-------- moses2/TranslationModel/MSPT/MSPT.cpp | 29 +++++++++++++++------------ moses2/TranslationModel/PhraseTable.cpp | 8 +------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp index 6e4190a4e..b13a648ee 100644 --- a/moses2/PhraseBased/SentenceWithCandidates.cpp +++ b/moses2/PhraseBased/SentenceWithCandidates.cpp @@ -38,10 +38,10 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, input_parts.push_back(copy_range(*It)); } - cerr << "Number of subparts: " << input_parts.size() << endl; + //cerr << "Number of subparts: " << input_parts.size() << endl; if (input_parts.size() ==2 ) { - cerr << "correct number of parts" << endl ; + //cerr << "correct number of parts" << endl ; } else { // TODO: how to handle wrong input format cerr << "INCORRECT number of parts" << endl ; @@ -50,8 +50,8 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, trim(input_parts[0]); trim(input_parts[1]); - cerr << "Input String: " << input_parts[0] << endl ; - cerr << "Phrase Table: " << input_parts[1] << endl ; + //cerr << "Input String: " << input_parts[0] << endl ; + //cerr << "Phrase Table: " << input_parts[1] << endl ; ///// Process the text part of the input const string partstr = input_parts[0]; @@ -72,7 +72,7 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n"); // ret->m_phraseTableString="constant phrase table"; // cerr << "Extracted Phrase Table String: " << ret->m_phraseTableString << endl; - cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl; + //cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl; return ret; } @@ -80,17 +80,17 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, SentenceWithCandidates::SentenceWithCandidates(MemPool &pool, size_t size) :Sentence(pool, size) { - cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl; + //cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl; } SentenceWithCandidates::~SentenceWithCandidates() { - cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl; + //cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl; } std::string SentenceWithCandidates::Debug(const System &system) const { - cerr << "SentenceWithCandidates::Debug" << endl; + return "SentenceWithCandidates::Debug"; } } /* namespace Moses2 */ diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index d72d0b395..0c20b2669 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -30,6 +30,7 @@ #include "../../SCFG/Manager.h" #include "../../PhraseBased/SentenceWithCandidates.h" +#include "../../PhraseBased/Manager.h" using namespace std; @@ -55,7 +56,7 @@ MSPT::~MSPT() void MSPT::CreatePTForInput(const System &system, string phraseTableString) { - cerr << "In CreatePTForInput" << endl << flush; + //cerr << "In CreatePTForInput" << endl << flush; FactorCollection &vocab = system.GetVocab(); MemPool &systemPool = system.GetSystemPool(); @@ -105,7 +106,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, *target); - //cerr << "EvaluateInIsolation:" << *target << endl; + //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl; m_rootPb->AddRule(m_input, *source, target); //cerr << "target=" << target->Debug(system) << endl; @@ -158,27 +159,29 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) void MSPT::InitializeForInput(const System &system, const InputType &input) { - cerr << "InitializeForInput MSPT" << endl; - cerr << input.Debug(system) << endl << flush; - cerr << "HH1" << endl << flush; + //cerr << "InitializeForInput MSPT" << endl; + //cerr << input.Debug(system) << endl; + //cerr << "HH1" << endl << flush; // downcast to SentenceWithCandidates - //const SentenceWithCandidates &inputObj = static_cast(input); - const SentenceWithCandidates &inputObj = dynamic_cast(input); - cerr << "Casting done." << endl << flush; - cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl; - - cerr << "Hardcoding sample PhraseTableString" << endl << flush; - string phraseTableString="a ||| x ||| 0.4 $$$ a ||| y ||| 0.6 $$$ b ||| y ||| 0.1 $$$ b ||| z ||| 0.9"; - CreatePTForInput(system,phraseTableString); + const SentenceWithCandidates &inputObj = static_cast(input); + //const SentenceWithCandidates &inputObj = dynamic_cast(input); + //cerr << "Casting done." << endl << flush; + //cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl << flush; + //cerr << "HH2" << endl << flush; + CreatePTForInput(system, inputObj.getPhraseTableString()); + //cerr << "HH3" << endl << flush; } TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, InputPath &inputPath) const { + //cerr << "MSPT::Lookup inputPath:" << inputPath.Debug(mgr.system) << endl; const SubPhrase &phrase = inputPath.subPhrase; TargetPhrases *tps = m_rootPb->Find(m_input, phrase); + //cerr << "MSPT::Lookup tps:" << tps->Debug(mgr.system) << endl; + //cerr << "MSPT::Lookup done" << endl; return tps; } diff --git a/moses2/TranslationModel/PhraseTable.cpp b/moses2/TranslationModel/PhraseTable.cpp index fef6771d6..1f87b8040 100644 --- a/moses2/TranslationModel/PhraseTable.cpp +++ b/moses2/TranslationModel/PhraseTable.cpp @@ -80,13 +80,7 @@ void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const if (SatisfyBackoff(mgr, *path)) { TargetPhrases *tpsPtr = Lookup(mgr, mgr.GetPool(), *path); - /* - cerr << "tpsPtr=" << tpsPtr << " "; - if (tps.get()) { - cerr << tps.get()->GetSize(); - } - cerr << endl; - */ + //cerr << "tpsPtr=" << tpsPtr << endl; path->AddTargetPhrases(*this, tpsPtr); } -- cgit v1.2.3 From cd18c7aa795db8b0f3e41a8dd52d3e2ba7bcd39d Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 5 Oct 2020 15:12:03 -0700 Subject: delete scfp --- moses2/TranslationModel/MSPT/MSPT.cpp | 34 +++------------------------------- moses2/TranslationModel/MSPT/MSPT.h | 1 - 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 0c20b2669..88e59324f 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -43,7 +43,6 @@ namespace Moses2 MSPT::MSPT(size_t startInd, const std::string &line) :PhraseTable(startInd, line) ,m_rootPb(NULL) - ,m_rootSCFG(NULL) { ReadParameters(); } @@ -51,7 +50,6 @@ MSPT::MSPT(size_t startInd, const std::string &line) MSPT::~MSPT() { delete m_rootPb; - delete m_rootSCFG; } void MSPT::CreatePTForInput(const System &system, string phraseTableString) @@ -65,7 +63,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) if (system.isPb) { m_rootPb = new PBNODE(); } else { - m_rootSCFG = new SCFGNODE(); + abort(); //cerr << "m_rootSCFG=" << m_rootSCFG << endl; } @@ -111,32 +109,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) //cerr << "target=" << target->Debug(system) << endl; } else { - SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); - //cerr << "created source:" << *source << endl; - SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, - system, toks[1]); - - //cerr << "created target " << *target << " source=" << *source << endl; - - target->GetScores().CreateFromString(toks[2], *this, system, true); - //cerr << "created scores:" << *target << endl; - - //vector scores = Tokenize(toks[2]); - //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; - - target->SetAlignmentInfo(toks[3]); - - // properties - if (toks.size() == 7) { - //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); - //strcpy(target->properties, toks[6].c_str()); - } - - system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, - *target); - //cerr << "EvaluateInIsolation:" << *target << endl; - m_rootSCFG->AddRule(m_input, *source, target); + abort(); } } @@ -144,8 +117,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); //cerr << "root=" << &m_rootPb << endl; } else { - m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); - //cerr << "root=" << &m_rootPb << endl; + abort(); } /* BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index 744158ea0..e527d9ead 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -69,7 +69,6 @@ public: protected: PBNODE *m_rootPb; - SCFGNODE *m_rootSCFG; void LookupGivenNode( MemPool &pool, -- cgit v1.2.3 From dd638e16f5af7ded5b82eb942fec3d06294046d7 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 5 Oct 2020 15:26:20 -0700 Subject: use manager pool instead of system pool. Get ready for thread-safe --- moses2/FF/FeatureFunction.h | 2 +- moses2/FF/FeatureFunctions.cpp | 4 ++-- moses2/FF/FeatureFunctions.h | 2 +- moses2/PhraseBased/Manager.cpp | 2 +- moses2/TranslationModel/MSPT/MSPT.cpp | 25 ++++++++----------------- moses2/TranslationModel/MSPT/MSPT.h | 4 ++-- 6 files changed, 15 insertions(+), 24 deletions(-) diff --git a/moses2/FF/FeatureFunction.h b/moses2/FF/FeatureFunction.h index 828a712b9..4fa2ee7c3 100644 --- a/moses2/FF/FeatureFunction.h +++ b/moses2/FF/FeatureFunction.h @@ -96,7 +96,7 @@ public: const SCFG::TargetPhrases &tps, const Phrase &sourcePhrase) const { } - virtual void InitializeForInput(const System &system, const InputType &input) { }; + virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input) { }; // clean up temporary memory, called after processing each sentence virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp index 39e2436b6..c11a2aadf 100644 --- a/moses2/FF/FeatureFunctions.cpp +++ b/moses2/FF/FeatureFunctions.cpp @@ -229,10 +229,10 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const } } -void FeatureFunctions::InitializeForInput(const InputType &input) +void FeatureFunctions::InitializeForInput(const ManagerBase &mgr, const InputType &input) { BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) { - ff->InitializeForInput(m_system, input); + ff->InitializeForInput(mgr, input); } } diff --git a/moses2/FF/FeatureFunctions.h b/moses2/FF/FeatureFunctions.h index 4c15d674a..43a5793c4 100644 --- a/moses2/FF/FeatureFunctions.h +++ b/moses2/FF/FeatureFunctions.h @@ -87,7 +87,7 @@ public: void EvaluateWhenAppliedBatch(const Batch &batch) const; - void InitializeForInput(const InputType &input); + void InitializeForInput(const ManagerBase &mgr, const InputType &input); void CleanUpAfterSentenceProcessing(const InputType &input) const; void ShowWeights(const Weights &allWeights); diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index cce30efb1..83cca53b1 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -63,7 +63,7 @@ void Manager::Init() //TODO: need option to choose Sentence vs SentenceWithCandidates m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); //cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush; - system.featureFunctions.InitializeForInput(*m_input); + system.featureFunctions.InitializeForInput(*this, *m_input); m_bitmaps = new Bitmaps(GetPool()); diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index 88e59324f..eb06fda21 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -52,12 +52,13 @@ MSPT::~MSPT() delete m_rootPb; } -void MSPT::CreatePTForInput(const System &system, string phraseTableString) +void MSPT::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) { //cerr << "In CreatePTForInput" << endl << flush; - + const System &system = mgr.system; FactorCollection &vocab = system.GetVocab(); MemPool &systemPool = system.GetSystemPool(); + MemPool &pool = mgr.GetPool(); MemPool tmpSourcePool; if (system.isPb) { @@ -85,7 +86,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[0]); //cerr << "created soure" << endl; - TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, system, toks[1]); //cerr << "created target" << endl; target->GetScores().CreateFromString(toks[2], *this, system, true); @@ -102,7 +103,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) //strcpy(target->properties, toks[6].c_str()); } - system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + system.featureFunctions.EvaluateInIsolation(pool, system, *source, *target); //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl; m_rootPb->AddRule(m_input, *source, target); @@ -114,7 +115,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) } if (system.isPb) { - m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); + m_rootPb->SortAndPrune(m_tableLimit, pool, system); //cerr << "root=" << &m_rootPb << endl; } else { abort(); @@ -129,21 +130,11 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString) } -void MSPT::InitializeForInput(const System &system, const InputType &input) +void MSPT::InitializeForInput(const ManagerBase &mgr, const InputType &input) { - //cerr << "InitializeForInput MSPT" << endl; - //cerr << input.Debug(system) << endl; - //cerr << "HH1" << endl << flush; - // downcast to SentenceWithCandidates const SentenceWithCandidates &inputObj = static_cast(input); - //const SentenceWithCandidates &inputObj = dynamic_cast(input); - //cerr << "Casting done." << endl << flush; - //cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl << flush; - //cerr << "HH2" << endl << flush; - CreatePTForInput(system, inputObj.getPhraseTableString()); - //cerr << "HH3" << endl << flush; - + CreatePTForInput(mgr, inputObj.getPhraseTableString()); } TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index e527d9ead..f13fe5847 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -65,7 +65,7 @@ public: const SCFG::Stacks &stacks, SCFG::InputPath &path) const; - virtual void InitializeForInput(const System &system, const InputType &input); + virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input); protected: PBNODE *m_rootPb; @@ -79,7 +79,7 @@ protected: const Moses2::Range &subPhraseRange, SCFG::InputPath &outPath) const; - void CreatePTForInput(const System &system, std::string phraseTableString); + void CreatePTForInput(const ManagerBase &mgr, std::string phraseTableString); }; -- cgit v1.2.3 From 74e2e11bfaab3920af808a8ffeab0be8e836f90b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 5 Oct 2020 15:39:26 -0700 Subject: thread-safe --- moses2/TranslationModel/MSPT/MSPT.cpp | 8 +++++--- moses2/TranslationModel/MSPT/MSPT.h | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp index eb06fda21..a30169f29 100644 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ b/moses2/TranslationModel/MSPT/MSPT.cpp @@ -36,13 +36,12 @@ using namespace std; namespace Moses2 { - +thread_local MSPT::PBNODE *MSPT::m_rootPb; //////////////////////////////////////////////////////////////////////// MSPT::MSPT(size_t startInd, const std::string &line) :PhraseTable(startInd, line) - ,m_rootPb(NULL) { ReadParameters(); } @@ -57,7 +56,6 @@ void MSPT::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) //cerr << "In CreatePTForInput" << endl << flush; const System &system = mgr.system; FactorCollection &vocab = system.GetVocab(); - MemPool &systemPool = system.GetSystemPool(); MemPool &pool = mgr.GetPool(); MemPool tmpSourcePool; @@ -148,6 +146,10 @@ TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, return tps; } +void MSPT::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { + delete m_rootPb; +} + void MSPT::InitActiveChart( MemPool &pool, const SCFG::Manager &mgr, diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h index f13fe5847..fe37e34b9 100644 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ b/moses2/TranslationModel/MSPT/MSPT.h @@ -66,9 +66,10 @@ public: SCFG::InputPath &path) const; virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input); + virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const; protected: - PBNODE *m_rootPb; + thread_local static PBNODE *m_rootPb; void LookupGivenNode( MemPool &pool, -- cgit v1.2.3 From a312312e7806942ea9535a5e96e5ec04e0d11f62 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 14 Oct 2020 19:00:25 +0530 Subject: windows build and managed clr --- .../other-builds/MosesManagedDLL/ManagedMoses.cpp | 116 ++++++++++++++ .../MosesManagedDLL/MosesManagedDLL.vcxproj | 170 ++++++++++++++++++++ .../MosesManagedDLL.vcxproj.filters | 22 +++ .../other-builds/moses2wrapper/ManagedMoses.cpp | 36 +++++ .../moses2wrapper/moses2wrapper.vcxproj | 176 +++++++++++++++++++++ moses2/Main.cpp | 6 +- moses2/Moses2Wrapper.cpp | 20 +++ moses2/Moses2Wrapper.h | 16 ++ moses2/TranslationTask.cpp | 11 ++ moses2/TranslationTask.h | 1 + moses2/server/Server.cpp | 4 +- 11 files changed, 573 insertions(+), 5 deletions(-) create mode 100644 contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp create mode 100644 contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj create mode 100644 contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters create mode 100644 contrib/other-builds/moses2wrapper/ManagedMoses.cpp create mode 100644 contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj create mode 100644 moses2/Moses2Wrapper.cpp create mode 100644 moses2/Moses2Wrapper.h diff --git a/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp new file mode 100644 index 000000000..0ba728360 --- /dev/null +++ b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp @@ -0,0 +1,116 @@ +#define NOMINMAX // Windows max macro collides with std:: +#include +#include +#include +#include +#include "Moses2Wrapper.h" + +using namespace System; +using namespace msclr::interop; + + +namespace Moses { + public ref class Moses2Wrapper + { + public: + Moses2Wrapper(String^ filePath) { + const std::string standardString = marshal_as(filePath); + m_pWrapper = new Moses2::Moses2Wrapper(standardString); + } + ~Moses2Wrapper() { this->!Moses2Wrapper(); } + String^ Translate(String^ input) { + const std::string standardString = marshal_as(input); + std::string output = m_pWrapper->Translate(standardString); + //Console::WriteLine(output); + String^ str = gcnew String(output.c_str()); + return str; + } + + protected: + !Moses2Wrapper() { delete m_pWrapper; m_pWrapper = nullptr; } + private: + Moses2::Moses2Wrapper *m_pWrapper; + }; +} +/* +public class ManagedMoses +{ + Moses2::Moses2Wrapper *m_Instance; +public: + ManagedMoses(String^ filepath) { + const std::string standardString = marshal_as(filepath); + m_Instance = new Moses2::Moses2Wrapper(standardString); + + } + String^ Translate(String^ input){ + const std::string standardString = marshal_as(input); + std::string output = m_Instance->Translate(standardString); + //Console::WriteLine(output); + String^ str = gcnew String(output.c_str()); + return str; + } +}; + + +/* +#include +#ifndef WIN32 +#define WIN32 +#endif +#include +#include "legacy/Parameter.h" +#include "System.h" + +using namespace System; +using namespace msclr::interop; + +// A wrapper around Faiss that lets you build indexes +// Right now just proof-of-concept code to makes sure it all works from C#, +// eventually may want to rework the interface, or possibly look at extending +// FaissSharp to support the windows dll + +namespace Moses { + + + + public ref class Parameter + { + public: + Parameter() { m_pWrapper = new Moses2::Parameter(); } + ~Parameter() { this->!Parameter(); } + bool LoadParams(String^ filePath) { + const std::string standardString = marshal_as(filePath); + auto flag = m_pWrapper->LoadParam(standardString); + return bool(flag); + } + Parameter* GetInstance() + { + return m_pWrapper; + } + protected: + !Parameter() { delete m_pWrapper; m_pWrapper = nullptr; } + private: + Moses2::Parameter* m_pWrapper; + }; + + + public ref class System { + public: + System(const Parameter^ paramsArg) { + new Moses2::System(paramsArg->GetInstance()); + } + ~System() { this->!System(); } + protected: + !System() { delete m_sWrapper; m_sWrapper = nullptr; } + private: + Moses2::System* m_sWrapper; + Moses2::Parameter* paramArgs; + }; + + + + + +} + +*/ \ No newline at end of file diff --git a/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj b/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj new file mode 100644 index 000000000..804bcc011 --- /dev/null +++ b/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj @@ -0,0 +1,170 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {5fb67144-27c9-4993-a084-e2097ffcc4fe} + MosesManagedDLL + 10.0 + + + + DynamicLibrary + true + v142 + Unicode + + + DynamicLibrary + false + v142 + true + Unicode + + + DynamicLibrary + true + v142 + Unicode + true + + + DynamicLibrary + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + false + + + true + + + false + + + + Level3 + true + WIN32;_DEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + false + + + + + Level3 + true + true + true + WIN32;NDEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + Level3 + true + _DEBUG;MOSESMANAGEDDLL_EXPORTS;NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_CONSOLE;_LIB;%(PreprocessorDefinitions) + false + NotUsing + pch.h + F:\boost_1_72_0;D:\moses-mstranslator;D:\xmlrpc-c-1.51.06\include;D:\moses-mstranslator\moses2 + true + MultiThreadedDebugDLL + + + Windows + true + false + D:\zlib-1.2.8\contrib\vstudio\vc11\x64\ZlibDllRelease;D:\xmlrpc-c-1.51.06\bin\Debug-Static-x64;C:\Users\sija\Downloads\2.0.2\bonitao-cmph-e5f83da\Debug;D:\zlib_1_2_8_msvc2015_64\msvc2015_64\lib\zlib;D:\mman-win32-master\x64\Release;F:\boost_1_72_0\lib64-msvc-14.2;%(AdditionalLibraryDirectories) + libxmlrpc.lib;libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;zlibwapi.lib;mman.lib;cmph.lib;%(AdditionalDependencies) + + + + + Level3 + true + true + true + NDEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + + + + {b4304e97-d37f-4022-bd03-841a4faee398} + + + + + + \ No newline at end of file diff --git a/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters b/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters new file mode 100644 index 000000000..8bd68daf0 --- /dev/null +++ b/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/contrib/other-builds/moses2wrapper/ManagedMoses.cpp b/contrib/other-builds/moses2wrapper/ManagedMoses.cpp new file mode 100644 index 000000000..adba8845f --- /dev/null +++ b/contrib/other-builds/moses2wrapper/ManagedMoses.cpp @@ -0,0 +1,36 @@ +#include +#include "legacy\Parameter.h" +#include "System.h" + +using namespace System; +using namespace msclr::interop; + +// A wrapper around Faiss that lets you build indexes +// Right now just proof-of-concept code to makes sure it all works from C#, +// eventually may want to rework the interface, or possibly look at extending +// FaissSharp to support the windows dll + +namespace Moses { + + public ref class System + { + + public: + + }; + + public ref class Parameter + { + public: + Parameter() { m_pWrapper = new Moses2::Parameter(); } + ~Parameter() { this->!Parameter(); } + + + + private: + // Review: I'm not using e.g. unique_ptr here because I don't know the lifetime + // semantics behind ref classes. + Moses2::Parameter* m_pWrapper; + }; + +} diff --git a/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj b/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj new file mode 100644 index 000000000..aa75c8dab --- /dev/null +++ b/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj @@ -0,0 +1,176 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {a190cd7d-ee0a-4eaa-8093-a751df1d4157} + moses2wrapper + 10.0 + + + + DynamicLibrary + true + v142 + Unicode + + + DynamicLibrary + false + v142 + true + Unicode + + + DynamicLibrary + true + v142 + Unicode + + + DynamicLibrary + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + false + + + true + + + false + + + + Level3 + true + WIN32;_DEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + false + + + + + Level3 + true + true + true + WIN32;NDEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + Level3 + true + _DEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + NotUsing + pch.h + true + ProgramDatabase + Classic + Async + Default + true + true + true + /Zc:twoPhase- %(AdditionalOptions) + F:\boost_1_72_0;D:\mman-win32-master;D:\zlib-1.2.8;D:\cmph-2.0.2;D:\moses-mstranslator;D:\xmlrpc-c-1.51.06\include;D:\moses-mstranslator\moses2 + MultiThreadedDebugDLL + + + Windows + true + false + F:\boost_1_72_0\lib64-msvc-14.2;%(AdditionalLibraryDirectories) + + + + + Level3 + true + true + true + NDEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + {b4304e97-d37f-4022-bd03-841a4faee398} + + + + + + + + + \ No newline at end of file diff --git a/moses2/Main.cpp b/moses2/Main.cpp index cf833760a..7054ec4ce 100644 --- a/moses2/Main.cpp +++ b/moses2/Main.cpp @@ -6,7 +6,7 @@ #include "Phrase.h" #include "TranslationTask.h" #include "MemPoolAllocator.h" -#include "server/Server.h" +//#include "server/Server.h" #include "legacy/InputFileStream.h" #include "legacy/Parameter.h" #include "legacy/ThreadPool.h" @@ -59,8 +59,8 @@ int main(int argc, char** argv) //////////////////////////////////////////////////////////////////////////////////////////////// void run_as_server(Moses2::System &system) { - Moses2::Server server(system.options.server, system); - server.run(system); // actually: don't return. see Server::run() + //Moses2::Server server(system.options.server, system); + //server.run(system); // actually: don't return. see Server::run() } //////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp new file mode 100644 index 000000000..2d7155e03 --- /dev/null +++ b/moses2/Moses2Wrapper.cpp @@ -0,0 +1,20 @@ +#include "Moses2Wrapper.h" +#include "System.h" +#include "legacy/Parameter.h" +#include "TranslationTask.h" +using namespace std; +namespace Moses2 { + Moses2Wrapper::Moses2Wrapper(const std::string &filePath) { + m_param = new Parameter(); + m_param->LoadParam(filePath); + m_system = new System(*m_param); + } + std::string Moses2Wrapper::Translate(const std::string &input) { + //create id + long a = 11234567; + TranslationTask task(*m_system, input, a); + std::string translation = task.RunTranslation(); + //delete translation; + return translation; + } +} \ No newline at end of file diff --git a/moses2/Moses2Wrapper.h b/moses2/Moses2Wrapper.h new file mode 100644 index 000000000..01d021145 --- /dev/null +++ b/moses2/Moses2Wrapper.h @@ -0,0 +1,16 @@ +#pragma once +#include +namespace Moses2 { + class Parameter; + class System; + class Moses2Wrapper + { + Parameter *m_param; + System *m_system; + + public: + Moses2Wrapper(const std::string &filePath); + std::string Translate(const std::string &input); + }; + +} \ No newline at end of file diff --git a/moses2/TranslationTask.cpp b/moses2/TranslationTask.cpp index 219d9ffcb..0032ab0b0 100644 --- a/moses2/TranslationTask.cpp +++ b/moses2/TranslationTask.cpp @@ -23,7 +23,18 @@ TranslationTask::TranslationTask(System &system, TranslationTask::~TranslationTask() { } +std::string TranslationTask::RunTranslation() +{ + m_mgr->Decode(); + + string out; + out = m_mgr->OutputBest() + "\n"; + + + delete m_mgr; + return out; +} void TranslationTask::Run() { diff --git a/moses2/TranslationTask.h b/moses2/TranslationTask.h index bf2330357..e683e1174 100644 --- a/moses2/TranslationTask.h +++ b/moses2/TranslationTask.h @@ -16,6 +16,7 @@ public: TranslationTask(System &system, const std::string &line, long translationId); virtual ~TranslationTask(); virtual void Run(); + virtual std::string RunTranslation(); protected: ManagerBase *m_mgr; diff --git a/moses2/server/Server.cpp b/moses2/server/Server.cpp index 57218c374..2da72270a 100644 --- a/moses2/server/Server.cpp +++ b/moses2/server/Server.cpp @@ -19,12 +19,12 @@ Server::Server(ServerOptions &server_options, System &system) :m_server_options(server_options) ,m_translator(new Translator(*this, system)) { - m_registry.addMethod("translate", m_translator); + // m_registry.addMethod("translate", m_translator); } Server::~Server() { - unlink(m_pidfile.c_str()); + //unlink(m_pidfile.c_str()); } void Server::run(System &system) -- cgit v1.2.3 From 8837f3c2a38906521f2ea2bfaebd54634945f48f Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Thu, 15 Oct 2020 15:15:35 +0530 Subject: code clean up --- moses2/Array.h | 2 +- moses2/InputType.cpp | 1 + moses2/Moses2Wrapper.cpp | 8 +++----- moses2/Moses2Wrapper.h | 2 +- moses2/PhraseBased/Manager.cpp | 11 +++++++---- moses2/TranslationTask.cpp | 6 +----- moses2/TranslationTask.h | 2 +- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/moses2/Array.h b/moses2/Array.h index d9402a704..fa6db557e 100644 --- a/moses2/Array.h +++ b/moses2/Array.h @@ -72,7 +72,7 @@ public: } void resize(size_t newSize) { - assert(m_size < m_maxSize); + assert(m_size <= m_maxSize); m_size = newSize; } protected: diff --git a/moses2/InputType.cpp b/moses2/InputType.cpp index af5b61ff9..889918a12 100644 --- a/moses2/InputType.cpp +++ b/moses2/InputType.cpp @@ -95,6 +95,7 @@ bool InputType::XmlOverlap(size_t startPos, size_t endPos) const std::string InputType::Debug(const System &system) const { cerr << "InputType::Debug" << endl; + return ""; } } /* namespace Moses2 */ diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index 2d7155e03..f919457d5 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -9,11 +9,9 @@ namespace Moses2 { m_param->LoadParam(filePath); m_system = new System(*m_param); } - std::string Moses2Wrapper::Translate(const std::string &input) { - //create id - long a = 11234567; - TranslationTask task(*m_system, input, a); - std::string translation = task.RunTranslation(); + std::string Moses2Wrapper::Translate(const std::string &input , long id) { + TranslationTask task(*m_system, input, id); + std::string translation = task.ReturnTranslation(); //delete translation; return translation; } diff --git a/moses2/Moses2Wrapper.h b/moses2/Moses2Wrapper.h index 01d021145..f491e1078 100644 --- a/moses2/Moses2Wrapper.h +++ b/moses2/Moses2Wrapper.h @@ -10,7 +10,7 @@ namespace Moses2 { public: Moses2Wrapper(const std::string &filePath); - std::string Translate(const std::string &input); + std::string Translate(const std::string &input, long id); }; } \ No newline at end of file diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp index 83cca53b1..a2a0ba0ad 100644 --- a/moses2/PhraseBased/Manager.cpp +++ b/moses2/PhraseBased/Manager.cpp @@ -60,9 +60,12 @@ void Manager::Init() InitPools(); FactorCollection &vocab = system.GetVocab(); - //TODO: need option to choose Sentence vs SentenceWithCandidates - m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); - //cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush; + if (system.options.input.input_type == SentenceInputWithCandidates) { + m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr); + } + else { + m_input = Moses2::Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr); + } system.featureFunctions.InitializeForInput(*this, *m_input); m_bitmaps = new Bitmaps(GetPool()); @@ -92,7 +95,7 @@ void Manager::Init() CalcFutureScore(); m_bitmaps->Init(sentence.GetSize(), vector(0)); - + switch (system.options.search.algo) { case Normal: m_search = new NSNormal::Search(*this); diff --git a/moses2/TranslationTask.cpp b/moses2/TranslationTask.cpp index 0032ab0b0..0644283b5 100644 --- a/moses2/TranslationTask.cpp +++ b/moses2/TranslationTask.cpp @@ -23,15 +23,11 @@ TranslationTask::TranslationTask(System &system, TranslationTask::~TranslationTask() { } -std::string TranslationTask::RunTranslation() +std::string TranslationTask::ReturnTranslation() const { m_mgr->Decode(); - string out; - out = m_mgr->OutputBest() + "\n"; - - delete m_mgr; return out; } diff --git a/moses2/TranslationTask.h b/moses2/TranslationTask.h index e683e1174..86e1766e1 100644 --- a/moses2/TranslationTask.h +++ b/moses2/TranslationTask.h @@ -16,7 +16,7 @@ public: TranslationTask(System &system, const std::string &line, long translationId); virtual ~TranslationTask(); virtual void Run(); - virtual std::string RunTranslation(); + virtual std::string ReturnTranslation() const; protected: ManagerBase *m_mgr; -- cgit v1.2.3 From 733418985d0a39e2c4a50963b305f247a2d20771 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Thu, 15 Oct 2020 15:17:21 +0530 Subject: adding test code --- .../other-builds/MosesManagedDLL/ManagedMoses.cpp | 122 +++------------------ .../MosesWrapperTest/MosesWrapperTest.csproj | 86 +++++++++++++++ contrib/other-builds/MosesWrapperTest/Program.cs | 20 ++++ 3 files changed, 123 insertions(+), 105 deletions(-) create mode 100644 contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj create mode 100644 contrib/other-builds/MosesWrapperTest/Program.cs diff --git a/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp index 0ba728360..da8b2393c 100644 --- a/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp +++ b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp @@ -1,7 +1,3 @@ -#define NOMINMAX // Windows max macro collides with std:: -#include -#include -#include #include #include "Moses2Wrapper.h" @@ -12,105 +8,21 @@ using namespace msclr::interop; namespace Moses { public ref class Moses2Wrapper { - public: - Moses2Wrapper(String^ filePath) { - const std::string standardString = marshal_as(filePath); - m_pWrapper = new Moses2::Moses2Wrapper(standardString); - } - ~Moses2Wrapper() { this->!Moses2Wrapper(); } - String^ Translate(String^ input) { - const std::string standardString = marshal_as(input); - std::string output = m_pWrapper->Translate(standardString); - //Console::WriteLine(output); - String^ str = gcnew String(output.c_str()); - return str; - } - - protected: - !Moses2Wrapper() { delete m_pWrapper; m_pWrapper = nullptr; } - private: - Moses2::Moses2Wrapper *m_pWrapper; + public: + Moses2Wrapper(String^ filePath) { + const std::string standardString = marshal_as(filePath); + m_pWrapper = new Moses2::Moses2Wrapper(standardString); + } + ~Moses2Wrapper() { this->!Moses2Wrapper(); } + String^ Translate(String^ input, long requestId) { + const std::string standardString = marshal_as(input); + std::string output = m_pWrapper->Translate(standardString, requestId); + String^ str = gcnew String(output.c_str()); + return str; + } + protected: + !Moses2Wrapper() { delete m_pWrapper; m_pWrapper = nullptr; } + private: + Moses2::Moses2Wrapper *m_pWrapper; }; -} -/* -public class ManagedMoses -{ - Moses2::Moses2Wrapper *m_Instance; -public: - ManagedMoses(String^ filepath) { - const std::string standardString = marshal_as(filepath); - m_Instance = new Moses2::Moses2Wrapper(standardString); - - } - String^ Translate(String^ input){ - const std::string standardString = marshal_as(input); - std::string output = m_Instance->Translate(standardString); - //Console::WriteLine(output); - String^ str = gcnew String(output.c_str()); - return str; - } -}; - - -/* -#include -#ifndef WIN32 -#define WIN32 -#endif -#include -#include "legacy/Parameter.h" -#include "System.h" - -using namespace System; -using namespace msclr::interop; - -// A wrapper around Faiss that lets you build indexes -// Right now just proof-of-concept code to makes sure it all works from C#, -// eventually may want to rework the interface, or possibly look at extending -// FaissSharp to support the windows dll - -namespace Moses { - - - - public ref class Parameter - { - public: - Parameter() { m_pWrapper = new Moses2::Parameter(); } - ~Parameter() { this->!Parameter(); } - bool LoadParams(String^ filePath) { - const std::string standardString = marshal_as(filePath); - auto flag = m_pWrapper->LoadParam(standardString); - return bool(flag); - } - Parameter* GetInstance() - { - return m_pWrapper; - } - protected: - !Parameter() { delete m_pWrapper; m_pWrapper = nullptr; } - private: - Moses2::Parameter* m_pWrapper; - }; - - - public ref class System { - public: - System(const Parameter^ paramsArg) { - new Moses2::System(paramsArg->GetInstance()); - } - ~System() { this->!System(); } - protected: - !System() { delete m_sWrapper; m_sWrapper = nullptr; } - private: - Moses2::System* m_sWrapper; - Moses2::Parameter* paramArgs; - }; - - - - - -} - -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj b/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj new file mode 100644 index 000000000..333fca400 --- /dev/null +++ b/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj @@ -0,0 +1,86 @@ + + + + + Debug + AnyCPU + {A2B603DB-F52E-4A83-BB61-F23D212E6482} + Exe + MosesWrapperTest + MosesWrapperTest + v4.7.2 + 512 + true + true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 2 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + {5fb67144-27c9-4993-a084-e2097ffcc4fe} + MosesManagedDLL + + + + + False + Microsoft .NET Framework 4.7.2 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + + + + \ No newline at end of file diff --git a/contrib/other-builds/MosesWrapperTest/Program.cs b/contrib/other-builds/MosesWrapperTest/Program.cs new file mode 100644 index 000000000..967262590 --- /dev/null +++ b/contrib/other-builds/MosesWrapperTest/Program.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Moses; +namespace MosesWrapperTest +{ + class Program { + + static void Main(string[] args) + { + Moses2Wrapper e = new Moses2Wrapper("D:/moses-mstranslator/test_sentence_with_candidates/moses_mspt.ini"); + long requestid = 123456; + string mystring = e.Translate("फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961", (int)requestid); + Console.WriteLine(mystring); + return; + } + } +} -- cgit v1.2.3 From 84fb13c57c446d30903a21426bea345979f5efa2 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Thu, 15 Oct 2020 15:22:15 +0530 Subject: nitpick --- contrib/other-builds/MosesWrapperTest/Program.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/other-builds/MosesWrapperTest/Program.cs b/contrib/other-builds/MosesWrapperTest/Program.cs index 967262590..97fed2863 100644 --- a/contrib/other-builds/MosesWrapperTest/Program.cs +++ b/contrib/other-builds/MosesWrapperTest/Program.cs @@ -11,8 +11,7 @@ namespace MosesWrapperTest static void Main(string[] args) { Moses2Wrapper e = new Moses2Wrapper("D:/moses-mstranslator/test_sentence_with_candidates/moses_mspt.ini"); - long requestid = 123456; - string mystring = e.Translate("फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961", (int)requestid); + string mystring = e.Translate("फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961", 123456789); Console.WriteLine(mystring); return; } -- cgit v1.2.3 From 48fa90931b40523143738a7aaf0a9b36be310a3e Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 16 Oct 2020 10:30:47 +0530 Subject: adding preprocessor for xmlrpc server , renaming MSPT to DynamicPT --- moses2/FF/FeatureRegistry.cpp | 4 +- moses2/Main.cpp | 76 +++++---- .../Dynamic/DynamicPhraseTable.cpp | 180 ++++++++++++++++++++ .../TranslationModel/Dynamic/DynamicPhraseTable.h | 88 ++++++++++ .../Dynamic/DynamicPhraseTableNode.h | 131 +++++++++++++++ moses2/TranslationModel/MSPT/MSNode.h | 131 --------------- moses2/TranslationModel/MSPT/MSPT.cpp | 183 --------------------- moses2/TranslationModel/MSPT/MSPT.h | 88 ---------- moses2/parameters/AllOptions.cpp | 4 +- moses2/parameters/AllOptions.h | 6 +- moses2/parameters/ServerOptions.cpp | 2 + moses2/parameters/ServerOptions.h | 5 + moses2/server/Server.cpp | 2 + moses2/server/TranslationRequest.cpp | 2 + moses2/server/Translator.cpp | 2 + 15 files changed, 468 insertions(+), 436 deletions(-) create mode 100644 moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp create mode 100644 moses2/TranslationModel/Dynamic/DynamicPhraseTable.h create mode 100644 moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h delete mode 100644 moses2/TranslationModel/MSPT/MSNode.h delete mode 100644 moses2/TranslationModel/MSPT/MSPT.cpp delete mode 100644 moses2/TranslationModel/MSPT/MSPT.h diff --git a/moses2/FF/FeatureRegistry.cpp b/moses2/FF/FeatureRegistry.cpp index 0ea56e6ce..8e457d371 100644 --- a/moses2/FF/FeatureRegistry.cpp +++ b/moses2/FF/FeatureRegistry.cpp @@ -4,7 +4,7 @@ #include "../TranslationModel/ProbingPT.h" #include "../TranslationModel/UnknownWordPenalty.h" #include "../TranslationModel/Transliteration.h" -#include "../TranslationModel/MSPT/MSPT.h" +#include "../TranslationModel/Dynamic/DynamicPhraseTable.h" #include "../LM/KENLM.h" #include "../LM/KENLMBatch.h" @@ -57,7 +57,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(ProbingPT); MOSES_FNAME2("PhraseDictionaryTransliteration", Transliteration); MOSES_FNAME(UnknownWordPenalty); - MOSES_FNAME(MSPT); + MOSES_FNAME(DynamicPhraseTable); Add("KENLM", new KenFactory()); diff --git a/moses2/Main.cpp b/moses2/Main.cpp index 7054ec4ce..626255566 100644 --- a/moses2/Main.cpp +++ b/moses2/Main.cpp @@ -6,7 +6,10 @@ #include "Phrase.h" #include "TranslationTask.h" #include "MemPoolAllocator.h" -//#include "server/Server.h" +#ifdef HAVE_SERVER + #include "server/Server.h" +#endif // HAVE_SERVER + #include "legacy/InputFileStream.h" #include "legacy/Parameter.h" #include "legacy/ThreadPool.h" @@ -38,17 +41,26 @@ int main(int argc, char** argv) } //cerr << "system.numThreads=" << system.options.server.numThreads << endl; - +#ifdef HAVE_SERVER Moses2::ThreadPool pool(system.options.server.numThreads, system.cpuAffinityOffset, system.cpuAffinityOffsetIncr); //cerr << "CREATED POOL" << endl; if (params.GetParam("server")) { std::cerr << "RUN SERVER" << std::endl; run_as_server(system); - } else { - std::cerr << "RUN BATCH" << std::endl; - batch_run(params, system, pool); } + else { + std::cerr << "RUN BATCH" << std::endl; + batch_run(params, system, pool); + } +#endif // +#ifndef HAVE_SERVER + Moses2::ThreadPool pool(15, system.cpuAffinityOffset, system.cpuAffinityOffsetIncr); + //cerr << "CREATED POOL" << endl; + + std::cerr << "RUN BATCH" << std::endl; + batch_run(params, system, pool); +#endif // !HAVE_SERVER cerr << "Decoding took " << timer.get_elapsed_time() << endl; // cerr << "g_numHypos=" << g_numHypos << endl; @@ -57,12 +69,14 @@ int main(int argc, char** argv) } //////////////////////////////////////////////////////////////////////////////////////////////// -void run_as_server(Moses2::System &system) -{ - //Moses2::Server server(system.options.server, system); - //server.run(system); // actually: don't return. see Server::run() -} - +#ifdef HAVE_SERVER + void run_as_server(Moses2::System& system) + { + Moses2::Server server(system.options.server, system); + server.run(system); // actually: don't return. see Server::run() + } + +#endif // HAVE_SERVER //////////////////////////////////////////////////////////////////////////////////////////////// istream &GetInputStream(Moses2::Parameter ¶ms) { @@ -76,31 +90,33 @@ istream &GetInputStream(Moses2::Parameter ¶ms) } //////////////////////////////////////////////////////////////////////////////////////////////// -void batch_run(Moses2::Parameter ¶ms, Moses2::System &system, Moses2::ThreadPool &pool) + +void batch_run(Moses2::Parameter& params, Moses2::System& system, Moses2::ThreadPool& pool) { - istream &inStream = GetInputStream(params); - - long translationId = 0; - string line; - while (getline(inStream, line)) { - //cerr << "line=" << line << endl; - boost::shared_ptr task(new Moses2::TranslationTask(system, line, translationId)); - - //cerr << "START pool.Submit()" << endl; - pool.Submit(task); - //task->Run(); - ++translationId; - } + istream& inStream = GetInputStream(params); - pool.Stop(true); + long translationId = 0; + string line; + while (getline(inStream, line)) { + //cerr << "line=" << line << endl; + boost::shared_ptr task(new Moses2::TranslationTask(system, line, translationId)); - if (&inStream != &cin) { - delete &inStream; - } + //cerr << "START pool.Submit()" << endl; + pool.Submit(task); + //task->Run(); + ++translationId; + } - //util::PrintUsage(std::cerr); + pool.Stop(true); + + if (&inStream != &cin) { + delete& inStream; + } + + //util::PrintUsage(std::cerr); } + //////////////////////////////////////////////////////////////////////////////////////////////// void Temp() { diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp new file mode 100644 index 000000000..0a74fcaa3 --- /dev/null +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp @@ -0,0 +1,180 @@ +/* + * DynamicPhraseTable.cpp + * + * Created on: 28 Oct 2015 + * Author: hieu + */ + +#include +#include +#include +#include "DynamicPhraseTable.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../Phrase.h" +#include "../../System.h" +#include "../../Scores.h" +#include "../../InputPathsBase.h" +#include "../../legacy/InputFileStream.h" +#include "util/exception.hh" + +#include "../../PhraseBased/InputPath.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../PhraseBased/SentenceWithCandidates.h" + +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/InputPath.h" +#include "../../SCFG/Stack.h" +#include "../../SCFG/Stacks.h" +#include "../../SCFG/Manager.h" + +#include "../../PhraseBased/SentenceWithCandidates.h" +#include "../../PhraseBased/Manager.h" + +using namespace std; + +namespace Moses2 +{ +thread_local DynamicPhraseTable::PBNODE * DynamicPhraseTable::m_rootPb; + +//////////////////////////////////////////////////////////////////////// + +DynamicPhraseTable::DynamicPhraseTable(size_t startInd, const std::string &line) + :PhraseTable(startInd, line) +{ + ReadParameters(); +} + +DynamicPhraseTable::~DynamicPhraseTable() +{ + delete m_rootPb; +} + +void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) +{ + //cerr << "In CreatePTForInput" << endl << flush; + const System &system = mgr.system; + FactorCollection &vocab = system.GetVocab(); + MemPool &pool = mgr.GetPool(); + MemPool tmpSourcePool; + + if (system.isPb) { + m_rootPb = new PBNODE(); + } else { + abort(); + //cerr << "m_rootSCFG=" << m_rootSCFG << endl; + } + + vector toks; + size_t lineNum = 0; + istringstream strme(phraseTableString); + string line; + while (getline(strme, line)) { + if (++lineNum % 1000000 == 0) { + cerr << lineNum << " "; + } + toks.clear(); + TokenizeMultiCharSeparator(toks, line, "|||"); + UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); + //cerr << "line=" << line << endl; + //cerr << "system.isPb=" << system.isPb << endl; + + if (system.isPb) { + PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created soure" << endl; + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, system, + toks[1]); + //cerr << "created target" << endl; + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + if (toks.size() >= 4) { + //cerr << "alignstr=" << toks[3] << endl; + target->SetAlignmentInfo(toks[3]); + } + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(pool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl; + m_rootPb->AddRule(m_input, *source, target); + + //cerr << "target=" << target->Debug(system) << endl; + } else { + abort(); + } + } + + if (system.isPb) { + m_rootPb->SortAndPrune(m_tableLimit, pool, system); + //cerr << "root=" << &m_rootPb << endl; + } else { + abort(); + } + /* + BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { + const Word &word = valPair.first; + cerr << word << " "; + } + cerr << endl; + */ + +} + +void DynamicPhraseTable::InitializeForInput(const ManagerBase &mgr, const InputType &input) +{ + // downcast to SentenceWithCandidates + const SentenceWithCandidates &inputObj = static_cast(input); + CreatePTForInput(mgr, inputObj.getPhraseTableString()); +} + +TargetPhrases* DynamicPhraseTable::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + const SubPhrase &phrase = inputPath.subPhrase; + TargetPhrases *tps = m_rootPb->Find(m_input, phrase); + return tps; +} + +void DynamicPhraseTable::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { + delete m_rootPb; +} + +void DynamicPhraseTable::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + abort(); +} + +void DynamicPhraseTable::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + abort(); +} + +void DynamicPhraseTable::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + abort(); +} + +} + diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h new file mode 100644 index 000000000..7fd467700 --- /dev/null +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h @@ -0,0 +1,88 @@ +/* + * MSPT.h + * + * Created on: 28 Oct 2015 + * Author: hieu + */ +#pragma once + +#include "../PhraseTable.h" +#include "../../legacy/Util2.h" +#include "../../SCFG/InputPath.h" +#include "DynamicPhraseTableNode.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/TargetPhrases.h" + +namespace Moses2 +{ + +class DynamicPhraseTable: public PhraseTable +{ + typedef DynamicPhraseTableNS::Node, TargetPhraseImpl, TargetPhrases> PBNODE; + typedef DynamicPhraseTableNS::Node, SCFG::TargetPhraseImpl, SCFG::TargetPhrases> SCFGNODE; + +////////////////////////////////////// + class ActiveChartEntryMem : public SCFG::ActiveChartEntry + { + typedef SCFG::ActiveChartEntry Parent; + public: + const DynamicPhraseTable::SCFGNODE &node; + + ActiveChartEntryMem(MemPool &pool, const DynamicPhraseTable::SCFGNODE &vnode) + :Parent(pool) + ,node(vnode) + {} + + ActiveChartEntryMem( + MemPool &pool, + const DynamicPhraseTable::SCFGNODE &vnode, + const ActiveChartEntry &prevEntry) + :Parent(prevEntry) + ,node(vnode) + {} + }; + + ////////////////////////////////////// +public: + DynamicPhraseTable(size_t startInd, const std::string &line); + virtual ~DynamicPhraseTable(); + + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + + virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input); + virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const; + +protected: + thread_local static PBNODE *m_rootPb; + + void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + void CreatePTForInput(const ManagerBase &mgr, std::string phraseTableString); + +}; + +} + diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h b/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h new file mode 100644 index 000000000..b10e4e161 --- /dev/null +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h @@ -0,0 +1,131 @@ +/* + * Node.h + * + * Created on: 22 Apr 2016 + * Author: hieu + */ +#pragma once +#include +#include +#include "../../PhraseBased/TargetPhrases.h" +#include "../../System.h" +#include "../../Phrase.h" + +namespace Moses2 +{ +class System; + +namespace DynamicPhraseTableNS +{ + +template +class Node +{ +public: + typedef boost::unordered_map Children; + + Node() + :m_targetPhrases(NULL) + ,m_unsortedTPS(NULL) + {} + + ~Node() + {} + + void AddRule(const std::vector &factors, SP &source, TP *target) { + AddRule(factors, source, target, 0); + } + + TPS *Find(const std::vector &factors, const SP &source, size_t pos = 0) const { + assert(source.GetSize()); + if (pos == source.GetSize()) { + return m_targetPhrases; + } else { + const WORD &word = source[pos]; + //cerr << "word=" << word << endl; + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } else { + const Node &child = iter->second; + return child.Find(factors, source, pos + 1); + } + } + } + + const Node *Find(const std::vector &factors, const WORD &word) const { + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } else { + const Node &child = iter->second; + return &child; + } + } + + const TPS *GetTargetPhrases() const { + return m_targetPhrases; + } + + void SortAndPrune(size_t tableLimit, MemPool &pool, const System &system) { + BOOST_FOREACH(typename Children::value_type &val, m_children) { + Node &child = val.second; + child.SortAndPrune(tableLimit, pool, system); + } + + // prune target phrases in this node + if (m_unsortedTPS) { + m_targetPhrases = new (pool.Allocate()) TPS(pool, m_unsortedTPS->size()); + + for (size_t i = 0; i < m_unsortedTPS->size(); ++i) { + TP *tp = (*m_unsortedTPS)[i]; + m_targetPhrases->AddTargetPhrase(*tp); + } + + m_targetPhrases->SortAndPrune(tableLimit); + system.featureFunctions.EvaluateAfterTablePruning(system.GetSystemPool(), *m_targetPhrases, *m_source); + + delete m_unsortedTPS; + } + } + + const Children &GetChildren() const { + return m_children; + } + + void Debug(std::ostream &out, const System &system) const { + BOOST_FOREACH(const typename Children::value_type &valPair, m_children) { + const WORD &word = valPair.first; + //std::cerr << word << "(" << word.hash() << ") "; + } + } +protected: + Children m_children; + TPS *m_targetPhrases; + Phrase *m_source; + std::vector *m_unsortedTPS; + + Node &AddRule(const std::vector &factors, SP &source, TP *target, size_t pos) { + if (pos == source.GetSize()) { + if (m_unsortedTPS == NULL) { + m_unsortedTPS = new std::vector(); + m_source = &source; + } + + m_unsortedTPS->push_back(target); + return *this; + } else { + const WORD &word = source[pos]; + Node &child = m_children[word.hash(factors)]; + //std::cerr << "added " << word << " " << &child << " from " << this << std::endl; + + return child.AddRule(factors, source, target, pos + 1); + } + } + +}; + + +} +} // namespace + diff --git a/moses2/TranslationModel/MSPT/MSNode.h b/moses2/TranslationModel/MSPT/MSNode.h deleted file mode 100644 index b02422aa5..000000000 --- a/moses2/TranslationModel/MSPT/MSNode.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Node.h - * - * Created on: 22 Apr 2016 - * Author: hieu - */ -#pragma once -#include -#include -#include "../../PhraseBased/TargetPhrases.h" -#include "../../System.h" -#include "../../Phrase.h" - -namespace Moses2 -{ -class System; - -namespace MSPTNS -{ - -template -class Node -{ -public: - typedef boost::unordered_map Children; - - Node() - :m_targetPhrases(NULL) - ,m_unsortedTPS(NULL) - {} - - ~Node() - {} - - void AddRule(const std::vector &factors, SP &source, TP *target) { - AddRule(factors, source, target, 0); - } - - TPS *Find(const std::vector &factors, const SP &source, size_t pos = 0) const { - assert(source.GetSize()); - if (pos == source.GetSize()) { - return m_targetPhrases; - } else { - const WORD &word = source[pos]; - //cerr << "word=" << word << endl; - typename Children::const_iterator iter = m_children.find(word.hash(factors)); - if (iter == m_children.end()) { - return NULL; - } else { - const Node &child = iter->second; - return child.Find(factors, source, pos + 1); - } - } - } - - const Node *Find(const std::vector &factors, const WORD &word) const { - typename Children::const_iterator iter = m_children.find(word.hash(factors)); - if (iter == m_children.end()) { - return NULL; - } else { - const Node &child = iter->second; - return &child; - } - } - - const TPS *GetTargetPhrases() const { - return m_targetPhrases; - } - - void SortAndPrune(size_t tableLimit, MemPool &pool, const System &system) { - BOOST_FOREACH(typename Children::value_type &val, m_children) { - Node &child = val.second; - child.SortAndPrune(tableLimit, pool, system); - } - - // prune target phrases in this node - if (m_unsortedTPS) { - m_targetPhrases = new (pool.Allocate()) TPS(pool, m_unsortedTPS->size()); - - for (size_t i = 0; i < m_unsortedTPS->size(); ++i) { - TP *tp = (*m_unsortedTPS)[i]; - m_targetPhrases->AddTargetPhrase(*tp); - } - - m_targetPhrases->SortAndPrune(tableLimit); - system.featureFunctions.EvaluateAfterTablePruning(system.GetSystemPool(), *m_targetPhrases, *m_source); - - delete m_unsortedTPS; - } - } - - const Children &GetChildren() const { - return m_children; - } - - void Debug(std::ostream &out, const System &system) const { - BOOST_FOREACH(const typename Children::value_type &valPair, m_children) { - const WORD &word = valPair.first; - //std::cerr << word << "(" << word.hash() << ") "; - } - } -protected: - Children m_children; - TPS *m_targetPhrases; - Phrase *m_source; - std::vector *m_unsortedTPS; - - Node &AddRule(const std::vector &factors, SP &source, TP *target, size_t pos) { - if (pos == source.GetSize()) { - if (m_unsortedTPS == NULL) { - m_unsortedTPS = new std::vector(); - m_source = &source; - } - - m_unsortedTPS->push_back(target); - return *this; - } else { - const WORD &word = source[pos]; - Node &child = m_children[word.hash(factors)]; - //std::cerr << "added " << word << " " << &child << " from " << this << std::endl; - - return child.AddRule(factors, source, target, pos + 1); - } - } - -}; - - -} -} // namespace - diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp deleted file mode 100644 index a30169f29..000000000 --- a/moses2/TranslationModel/MSPT/MSPT.cpp +++ /dev/null @@ -1,183 +0,0 @@ -/* - * MSPT.cpp - * - * Created on: 28 Oct 2015 - * Author: hieu - */ - -#include -#include -#include -#include "MSPT.h" -#include "../../PhraseBased/PhraseImpl.h" -#include "../../Phrase.h" -#include "../../System.h" -#include "../../Scores.h" -#include "../../InputPathsBase.h" -#include "../../legacy/InputFileStream.h" -#include "util/exception.hh" - -#include "../../PhraseBased/InputPath.h" -#include "../../PhraseBased/TargetPhraseImpl.h" -#include "../../PhraseBased/TargetPhrases.h" -#include "../../PhraseBased/SentenceWithCandidates.h" - -#include "../../SCFG/PhraseImpl.h" -#include "../../SCFG/TargetPhraseImpl.h" -#include "../../SCFG/InputPath.h" -#include "../../SCFG/Stack.h" -#include "../../SCFG/Stacks.h" -#include "../../SCFG/Manager.h" - -#include "../../PhraseBased/SentenceWithCandidates.h" -#include "../../PhraseBased/Manager.h" - -using namespace std; - -namespace Moses2 -{ -thread_local MSPT::PBNODE *MSPT::m_rootPb; - -//////////////////////////////////////////////////////////////////////// - -MSPT::MSPT(size_t startInd, const std::string &line) - :PhraseTable(startInd, line) -{ - ReadParameters(); -} - -MSPT::~MSPT() -{ - delete m_rootPb; -} - -void MSPT::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) -{ - //cerr << "In CreatePTForInput" << endl << flush; - const System &system = mgr.system; - FactorCollection &vocab = system.GetVocab(); - MemPool &pool = mgr.GetPool(); - MemPool tmpSourcePool; - - if (system.isPb) { - m_rootPb = new PBNODE(); - } else { - abort(); - //cerr << "m_rootSCFG=" << m_rootSCFG << endl; - } - - vector toks; - size_t lineNum = 0; - istringstream strme(phraseTableString); - string line; - while (getline(strme, line)) { - if (++lineNum % 1000000 == 0) { - cerr << lineNum << " "; - } - toks.clear(); - TokenizeMultiCharSeparator(toks, line, "|||"); - UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); - //cerr << "line=" << line << endl; - //cerr << "system.isPb=" << system.isPb << endl; - - if (system.isPb) { - PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); - //cerr << "created soure" << endl; - TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, system, - toks[1]); - //cerr << "created target" << endl; - target->GetScores().CreateFromString(toks[2], *this, system, true); - //cerr << "created scores:" << *target << endl; - - if (toks.size() >= 4) { - //cerr << "alignstr=" << toks[3] << endl; - target->SetAlignmentInfo(toks[3]); - } - - // properties - if (toks.size() == 7) { - //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); - //strcpy(target->properties, toks[6].c_str()); - } - - system.featureFunctions.EvaluateInIsolation(pool, system, *source, - *target); - //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl; - m_rootPb->AddRule(m_input, *source, target); - - //cerr << "target=" << target->Debug(system) << endl; - } else { - abort(); - } - } - - if (system.isPb) { - m_rootPb->SortAndPrune(m_tableLimit, pool, system); - //cerr << "root=" << &m_rootPb << endl; - } else { - abort(); - } - /* - BOOST_FOREACH(const PtMem::Node::Children::value_type &valPair, m_rootPb.GetChildren()) { - const Word &word = valPair.first; - cerr << word << " "; - } - cerr << endl; - */ - -} - -void MSPT::InitializeForInput(const ManagerBase &mgr, const InputType &input) -{ - // downcast to SentenceWithCandidates - const SentenceWithCandidates &inputObj = static_cast(input); - CreatePTForInput(mgr, inputObj.getPhraseTableString()); -} - -TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const -{ - //cerr << "MSPT::Lookup inputPath:" << inputPath.Debug(mgr.system) << endl; - const SubPhrase &phrase = inputPath.subPhrase; - TargetPhrases *tps = m_rootPb->Find(m_input, phrase); - //cerr << "MSPT::Lookup tps:" << tps->Debug(mgr.system) << endl; - //cerr << "MSPT::Lookup done" << endl; - return tps; -} - -void MSPT::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { - delete m_rootPb; -} - -void MSPT::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const -{ - abort(); -} - -void MSPT::Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const -{ - abort(); -} - -void MSPT::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const -{ - abort(); -} - -} - diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h deleted file mode 100644 index fe37e34b9..000000000 --- a/moses2/TranslationModel/MSPT/MSPT.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * MSPT.h - * - * Created on: 28 Oct 2015 - * Author: hieu - */ -#pragma once - -#include "../PhraseTable.h" -#include "../../legacy/Util2.h" -#include "../../SCFG/InputPath.h" -#include "MSNode.h" -#include "../../PhraseBased/PhraseImpl.h" -#include "../../PhraseBased/TargetPhraseImpl.h" -#include "../../PhraseBased/TargetPhrases.h" -#include "../../SCFG/PhraseImpl.h" -#include "../../SCFG/TargetPhraseImpl.h" -#include "../../SCFG/TargetPhrases.h" - -namespace Moses2 -{ - -class MSPT: public PhraseTable -{ - typedef MSPTNS::Node, TargetPhraseImpl, TargetPhrases> PBNODE; - typedef MSPTNS::Node, SCFG::TargetPhraseImpl, SCFG::TargetPhrases> SCFGNODE; - -////////////////////////////////////// - class ActiveChartEntryMem : public SCFG::ActiveChartEntry - { - typedef SCFG::ActiveChartEntry Parent; - public: - const MSPT::SCFGNODE &node; - - ActiveChartEntryMem(MemPool &pool, const MSPT::SCFGNODE &vnode) - :Parent(pool) - ,node(vnode) - {} - - ActiveChartEntryMem( - MemPool &pool, - const MSPT::SCFGNODE &vnode, - const ActiveChartEntry &prevEntry) - :Parent(prevEntry) - ,node(vnode) - {} - }; - - ////////////////////////////////////// -public: - MSPT(size_t startInd, const std::string &line); - virtual ~MSPT(); - - virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; - - virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; - - void Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; - - virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input); - virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const; - -protected: - thread_local static PBNODE *m_rootPb; - - void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; - - void CreatePTForInput(const ManagerBase &mgr, std::string phraseTableString); - -}; - -} - diff --git a/moses2/parameters/AllOptions.cpp b/moses2/parameters/AllOptions.cpp index 954d7e8e7..595f6b415 100644 --- a/moses2/parameters/AllOptions.cpp +++ b/moses2/parameters/AllOptions.cpp @@ -31,7 +31,9 @@ init(Parameter const& param) if (!lmbr.init(param)) return false; if (!output.init(param)) return false; if (!unk.init(param)) return false; +#if HAVE_SERVER if (!server.init(param)) return false; +#endif // HAVE_SERVER if (!syntax.init(param)) return false; param.SetParameter(mira, "mira", false); @@ -95,7 +97,7 @@ update(std::mapconst& param) if (!lmbr.update(param)) return false; if (!output.update(param)) return false; if (!unk.update(param)) return false; - if (!server.update(param)) return false; + //if (!server.update(param)) return false; //if (!syntax.update(param)) return false; return sanity_check(); } diff --git a/moses2/parameters/AllOptions.h b/moses2/parameters/AllOptions.h index 2f09cd385..187ddb7bb 100644 --- a/moses2/parameters/AllOptions.h +++ b/moses2/parameters/AllOptions.h @@ -13,7 +13,9 @@ #include "LMBR_Options.h" #include "ReportingOptions.h" #include "OOVHandlingOptions.h" -#include "ServerOptions.h" +#ifdef HAVE_SERVER + #include "ServerOptions.h" +#endif // HAVE_SERVER #include "SyntaxOptions.h" namespace Moses2 @@ -31,7 +33,9 @@ struct LMBR_Options lmbr; ReportingOptions output; OOVHandlingOptions unk; +#ifdef HAVE_SERVER ServerOptions server; +#endif // HAVE_SERVER SyntaxOptions syntax; bool mira; bool use_legacy_pt; diff --git a/moses2/parameters/ServerOptions.cpp b/moses2/parameters/ServerOptions.cpp index 3a21c1891..c00a43d26 100644 --- a/moses2/parameters/ServerOptions.cpp +++ b/moses2/parameters/ServerOptions.cpp @@ -1,4 +1,5 @@ // -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*- +#ifdef HAVE_SERVER #include #include #include "../legacy/Parameter.h" @@ -81,3 +82,4 @@ init(Parameter const& P) return true; } } // namespace Moses +#endif diff --git a/moses2/parameters/ServerOptions.h b/moses2/parameters/ServerOptions.h index 377b4d31b..39b109674 100644 --- a/moses2/parameters/ServerOptions.h +++ b/moses2/parameters/ServerOptions.h @@ -1,5 +1,9 @@ // -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*- #pragma once +#ifdef HAVE_SERVER + + + #include #include #include @@ -39,3 +43,4 @@ struct }; } +#endif // HAVE_SERVER diff --git a/moses2/server/Server.cpp b/moses2/server/Server.cpp index 2da72270a..e8baeadb3 100644 --- a/moses2/server/Server.cpp +++ b/moses2/server/Server.cpp @@ -4,6 +4,7 @@ * Created on: 1 Apr 2016 * Author: hieu */ +#ifdef HAVE_SERVER #include #include "../System.h" #include "Server.h" @@ -70,3 +71,4 @@ ServerOptions const&Server::options() const } /* namespace Moses2 */ +#endif // HAVE_SERVER diff --git a/moses2/server/TranslationRequest.cpp b/moses2/server/TranslationRequest.cpp index 2d50835a6..91ae3357f 100644 --- a/moses2/server/TranslationRequest.cpp +++ b/moses2/server/TranslationRequest.cpp @@ -1,3 +1,4 @@ +#ifdef HAVE_SERVER #include #include "TranslationRequest.h" #include "../ManagerBase.h" @@ -66,3 +67,4 @@ void TranslationRequest::pack_hypothesis(const Manager& manager, Hypothesis cons } } +#endif // HAVE_SERVER diff --git a/moses2/server/Translator.cpp b/moses2/server/Translator.cpp index 6f6212323..9d6dcbc85 100644 --- a/moses2/server/Translator.cpp +++ b/moses2/server/Translator.cpp @@ -4,6 +4,7 @@ * Created on: 1 Apr 2016 * Author: hieu */ +#ifdef HAVE_SERVER #include #include "Translator.h" #include "TranslationRequest.h" @@ -66,3 +67,4 @@ void Translator::execute(xmlrpc_c::paramList const& paramList, } } /* namespace Moses2 */ +#endif // HAVE_SERVER -- cgit v1.2.3 From 714a873e4b10955b4b64685a62e9e163687c84f2 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 16 Oct 2020 11:16:45 +0530 Subject: nitpick , uncomment the server and registry function --- moses2/parameters/ServerOptions.h | 2 -- moses2/server/Server.cpp | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/moses2/parameters/ServerOptions.h b/moses2/parameters/ServerOptions.h index 39b109674..49832abd6 100644 --- a/moses2/parameters/ServerOptions.h +++ b/moses2/parameters/ServerOptions.h @@ -2,8 +2,6 @@ #pragma once #ifdef HAVE_SERVER - - #include #include #include diff --git a/moses2/server/Server.cpp b/moses2/server/Server.cpp index e8baeadb3..66e83e0b7 100644 --- a/moses2/server/Server.cpp +++ b/moses2/server/Server.cpp @@ -20,12 +20,12 @@ Server::Server(ServerOptions &server_options, System &system) :m_server_options(server_options) ,m_translator(new Translator(*this, system)) { - // m_registry.addMethod("translate", m_translator); + m_registry.addMethod("translate", m_translator); } Server::~Server() { - //unlink(m_pidfile.c_str()); + unlink(m_pidfile.c_str()); } void Server::run(System &system) -- cgit v1.2.3 From c33c8d660536043318c35b2f5405fbaeb02e22ca Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 16 Oct 2020 11:17:49 +0530 Subject: TODO --- moses2/Main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moses2/Main.cpp b/moses2/Main.cpp index 626255566..07271b175 100644 --- a/moses2/Main.cpp +++ b/moses2/Main.cpp @@ -54,7 +54,9 @@ int main(int argc, char** argv) batch_run(params, system, pool); } #endif // +//TODO : WIN32 #ifndef HAVE_SERVER + // TODO : remove hardcoding for num of threads Moses2::ThreadPool pool(15, system.cpuAffinityOffset, system.cpuAffinityOffsetIncr); //cerr << "CREATED POOL" << endl; -- cgit v1.2.3 From baef2262001fc8fc894ed222643cbeb30fc9fe71 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 16 Oct 2020 11:23:46 +0530 Subject: todo --- contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp index da8b2393c..f897625c4 100644 --- a/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp +++ b/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp @@ -4,7 +4,7 @@ using namespace System; using namespace msclr::interop; - +//TODO: include headers as per the build process namespace Moses { public ref class Moses2Wrapper { -- cgit v1.2.3 From 44352a5315e036e9210cfc37a453d786edf15851 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Fri, 16 Oct 2020 11:26:15 +0530 Subject: removing unused code --- .../other-builds/moses2wrapper/ManagedMoses.cpp | 36 ----- .../moses2wrapper/moses2wrapper.vcxproj | 176 --------------------- 2 files changed, 212 deletions(-) delete mode 100644 contrib/other-builds/moses2wrapper/ManagedMoses.cpp delete mode 100644 contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj diff --git a/contrib/other-builds/moses2wrapper/ManagedMoses.cpp b/contrib/other-builds/moses2wrapper/ManagedMoses.cpp deleted file mode 100644 index adba8845f..000000000 --- a/contrib/other-builds/moses2wrapper/ManagedMoses.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include "legacy\Parameter.h" -#include "System.h" - -using namespace System; -using namespace msclr::interop; - -// A wrapper around Faiss that lets you build indexes -// Right now just proof-of-concept code to makes sure it all works from C#, -// eventually may want to rework the interface, or possibly look at extending -// FaissSharp to support the windows dll - -namespace Moses { - - public ref class System - { - - public: - - }; - - public ref class Parameter - { - public: - Parameter() { m_pWrapper = new Moses2::Parameter(); } - ~Parameter() { this->!Parameter(); } - - - - private: - // Review: I'm not using e.g. unique_ptr here because I don't know the lifetime - // semantics behind ref classes. - Moses2::Parameter* m_pWrapper; - }; - -} diff --git a/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj b/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj deleted file mode 100644 index aa75c8dab..000000000 --- a/contrib/other-builds/moses2wrapper/moses2wrapper.vcxproj +++ /dev/null @@ -1,176 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 16.0 - Win32Proj - {a190cd7d-ee0a-4eaa-8093-a751df1d4157} - moses2wrapper - 10.0 - - - - DynamicLibrary - true - v142 - Unicode - - - DynamicLibrary - false - v142 - true - Unicode - - - DynamicLibrary - true - v142 - Unicode - - - DynamicLibrary - false - v142 - true - Unicode - - - - - - - - - - - - - - - - - - - - - true - - - false - - - true - - - false - - - - Level3 - true - WIN32;_DEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - true - Use - pch.h - - - Windows - true - false - - - - - Level3 - true - true - true - WIN32;NDEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - true - Use - pch.h - - - Windows - true - true - true - false - - - - - Level3 - true - _DEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - NotUsing - pch.h - true - ProgramDatabase - Classic - Async - Default - true - true - true - /Zc:twoPhase- %(AdditionalOptions) - F:\boost_1_72_0;D:\mman-win32-master;D:\zlib-1.2.8;D:\cmph-2.0.2;D:\moses-mstranslator;D:\xmlrpc-c-1.51.06\include;D:\moses-mstranslator\moses2 - MultiThreadedDebugDLL - - - Windows - true - false - F:\boost_1_72_0\lib64-msvc-14.2;%(AdditionalLibraryDirectories) - - - - - Level3 - true - true - true - NDEBUG;MOSES2WRAPPER_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - true - Use - pch.h - - - Windows - true - true - true - false - - - - - {b4304e97-d37f-4022-bd03-841a4faee398} - - - - - - - - - \ No newline at end of file -- cgit v1.2.3 From 8dffc2da055a45faa08d26da9663cce606e6c182 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 15 Oct 2020 23:16:13 -0700 Subject: rename MSPT -> DynamicPhraseTable --- moses2/Jamfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moses2/Jamfile b/moses2/Jamfile index cf20046c3..022d4dcd6 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -84,7 +84,7 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp TranslationModel/CompactPT/ThrowingFwrite.cpp - TranslationModel/MSPT/MSPT.cpp + TranslationModel/Dynamic/DynamicPhraseTable.cpp parameters/AllOptions.cpp parameters/BookkeepingOptions.cpp -- cgit v1.2.3 From 47129bd4bcbacd36d358d43039eaf641f2231b35 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 16 Oct 2020 00:27:10 -0700 Subject: always compile moses2. No need to include xmlrpc-c --- moses2/Jamfile | 26 ++++++++++++++------------ moses2/Main.cpp | 32 ++++++++++++-------------------- moses2/parameters/AllOptions.cpp | 2 -- moses2/parameters/AllOptions.h | 6 +----- moses2/parameters/ServerOptions.cpp | 2 -- moses2/parameters/ServerOptions.h | 6 +----- moses2/server/Server.cpp | 2 -- moses2/server/TranslationRequest.cpp | 2 -- moses2/server/Translator.cpp | 2 -- 9 files changed, 28 insertions(+), 52 deletions(-) diff --git a/moses2/Jamfile b/moses2/Jamfile index 022d4dcd6..9c1a6840b 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -9,6 +9,17 @@ else { alias cmph ; } +if [ xmlrpc ] +{ + echo "BUILDING MOSES2 SERVER!" ; + alias mserver2 : [ glob server/*.cpp ] ; +} +else +{ + echo "NOT BUILDING MOSES2 SERVER!" ; + alias mserver2 ; +} + max-factors = [ option.get "max-factors" : 4 : 4 ] ; max-factors = MAX_NUM_FACTORS=$(max-factors) $(FACTOR-LOG) ; @@ -170,25 +181,16 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors SCFG/nbest/NBest.cpp SCFG/nbest/NBests.cpp SCFG/nbest/NBestColl.cpp - - server/Server.cpp - server/Translator.cpp - server/TranslationRequest.cpp deps cmph + mserver2 : $(includes) ; exe moses2 : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm ; -if [ xmlrpc ] { - echo "Building Moses2" ; - alias programs : moses2 ; -} -else { - echo "Not building Moses2" ; - alias programs : ; -} +echo "Building Moses2" ; +alias programs : moses2 ; diff --git a/moses2/Main.cpp b/moses2/Main.cpp index 07271b175..6fa1f5bd8 100644 --- a/moses2/Main.cpp +++ b/moses2/Main.cpp @@ -6,9 +6,9 @@ #include "Phrase.h" #include "TranslationTask.h" #include "MemPoolAllocator.h" -#ifdef HAVE_SERVER +#ifdef HAVE_XMLRPC_C #include "server/Server.h" -#endif // HAVE_SERVER +#endif // HAVE_XMLRPC_C #include "legacy/InputFileStream.h" #include "legacy/Parameter.h" @@ -41,7 +41,6 @@ int main(int argc, char** argv) } //cerr << "system.numThreads=" << system.options.server.numThreads << endl; -#ifdef HAVE_SERVER Moses2::ThreadPool pool(system.options.server.numThreads, system.cpuAffinityOffset, system.cpuAffinityOffsetIncr); //cerr << "CREATED POOL" << endl; @@ -53,16 +52,6 @@ int main(int argc, char** argv) std::cerr << "RUN BATCH" << std::endl; batch_run(params, system, pool); } -#endif // -//TODO : WIN32 -#ifndef HAVE_SERVER - // TODO : remove hardcoding for num of threads - Moses2::ThreadPool pool(15, system.cpuAffinityOffset, system.cpuAffinityOffsetIncr); - //cerr << "CREATED POOL" << endl; - - std::cerr << "RUN BATCH" << std::endl; - batch_run(params, system, pool); -#endif // !HAVE_SERVER cerr << "Decoding took " << timer.get_elapsed_time() << endl; // cerr << "g_numHypos=" << g_numHypos << endl; @@ -71,14 +60,17 @@ int main(int argc, char** argv) } //////////////////////////////////////////////////////////////////////////////////////////////// -#ifdef HAVE_SERVER - void run_as_server(Moses2::System& system) - { - Moses2::Server server(system.options.server, system); - server.run(system); // actually: don't return. see Server::run() - } +void run_as_server(Moses2::System& system) +{ +#ifdef HAVE_XMLRPC_C + Moses2::Server server(system.options.server, system); + server.run(system); // actually: don't return. see Server::run() +#else + UTIL_THROW2("Moses2 was compiled without xmlrpc-c. " + << "No server functionality available."); +#endif +} -#endif // HAVE_SERVER //////////////////////////////////////////////////////////////////////////////////////////////// istream &GetInputStream(Moses2::Parameter ¶ms) { diff --git a/moses2/parameters/AllOptions.cpp b/moses2/parameters/AllOptions.cpp index 595f6b415..664b8dc6d 100644 --- a/moses2/parameters/AllOptions.cpp +++ b/moses2/parameters/AllOptions.cpp @@ -31,9 +31,7 @@ init(Parameter const& param) if (!lmbr.init(param)) return false; if (!output.init(param)) return false; if (!unk.init(param)) return false; -#if HAVE_SERVER if (!server.init(param)) return false; -#endif // HAVE_SERVER if (!syntax.init(param)) return false; param.SetParameter(mira, "mira", false); diff --git a/moses2/parameters/AllOptions.h b/moses2/parameters/AllOptions.h index 187ddb7bb..2f09cd385 100644 --- a/moses2/parameters/AllOptions.h +++ b/moses2/parameters/AllOptions.h @@ -13,9 +13,7 @@ #include "LMBR_Options.h" #include "ReportingOptions.h" #include "OOVHandlingOptions.h" -#ifdef HAVE_SERVER - #include "ServerOptions.h" -#endif // HAVE_SERVER +#include "ServerOptions.h" #include "SyntaxOptions.h" namespace Moses2 @@ -33,9 +31,7 @@ struct LMBR_Options lmbr; ReportingOptions output; OOVHandlingOptions unk; -#ifdef HAVE_SERVER ServerOptions server; -#endif // HAVE_SERVER SyntaxOptions syntax; bool mira; bool use_legacy_pt; diff --git a/moses2/parameters/ServerOptions.cpp b/moses2/parameters/ServerOptions.cpp index c00a43d26..3a21c1891 100644 --- a/moses2/parameters/ServerOptions.cpp +++ b/moses2/parameters/ServerOptions.cpp @@ -1,5 +1,4 @@ // -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*- -#ifdef HAVE_SERVER #include #include #include "../legacy/Parameter.h" @@ -82,4 +81,3 @@ init(Parameter const& P) return true; } } // namespace Moses -#endif diff --git a/moses2/parameters/ServerOptions.h b/moses2/parameters/ServerOptions.h index 49832abd6..2b67e5156 100644 --- a/moses2/parameters/ServerOptions.h +++ b/moses2/parameters/ServerOptions.h @@ -1,13 +1,10 @@ // -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*- #pragma once -#ifdef HAVE_SERVER +#include "../legacy/xmlrpc-c.h" #include #include #include -#include -#include -#include namespace Moses2 { @@ -41,4 +38,3 @@ struct }; } -#endif // HAVE_SERVER diff --git a/moses2/server/Server.cpp b/moses2/server/Server.cpp index 66e83e0b7..de3542eb6 100644 --- a/moses2/server/Server.cpp +++ b/moses2/server/Server.cpp @@ -4,7 +4,6 @@ * Created on: 1 Apr 2016 * Author: hieu */ -#ifdef HAVE_SERVER #include #include "../System.h" #include "Server.h" @@ -71,4 +70,3 @@ ServerOptions const&Server::options() const } /* namespace Moses2 */ -#endif // HAVE_SERVER diff --git a/moses2/server/TranslationRequest.cpp b/moses2/server/TranslationRequest.cpp index 91ae3357f..2d50835a6 100644 --- a/moses2/server/TranslationRequest.cpp +++ b/moses2/server/TranslationRequest.cpp @@ -1,4 +1,3 @@ -#ifdef HAVE_SERVER #include #include "TranslationRequest.h" #include "../ManagerBase.h" @@ -67,4 +66,3 @@ void TranslationRequest::pack_hypothesis(const Manager& manager, Hypothesis cons } } -#endif // HAVE_SERVER diff --git a/moses2/server/Translator.cpp b/moses2/server/Translator.cpp index 9d6dcbc85..6f6212323 100644 --- a/moses2/server/Translator.cpp +++ b/moses2/server/Translator.cpp @@ -4,7 +4,6 @@ * Created on: 1 Apr 2016 * Author: hieu */ -#ifdef HAVE_SERVER #include #include "Translator.h" #include "TranslationRequest.h" @@ -67,4 +66,3 @@ void Translator::execute(xmlrpc_c::paramList const& paramList, } } /* namespace Moses2 */ -#endif // HAVE_SERVER -- cgit v1.2.3 From 3cc2f4fde68dffee9904cbee4dc4910b38c3f530 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Mon, 26 Oct 2020 14:06:23 +0530 Subject: Dynamic loading --- contrib/other-builds/dllImpCheck/Program.cs | 38 ++ contrib/other-builds/moses2/moses2.vcxproj.filters | 612 +++++++++++---------- moses2/DLLEntryApi.cpp | 39 ++ moses2/Moses2Wrapper.cpp | 10 +- moses2/Moses2Wrapper.h | 6 +- 5 files changed, 400 insertions(+), 305 deletions(-) create mode 100644 contrib/other-builds/dllImpCheck/Program.cs create mode 100644 moses2/DLLEntryApi.cpp diff --git a/contrib/other-builds/dllImpCheck/Program.cs b/contrib/other-builds/dllImpCheck/Program.cs new file mode 100644 index 000000000..4fbd38696 --- /dev/null +++ b/contrib/other-builds/dllImpCheck/Program.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; + +namespace dllImpCheck +{ + class Program + { + + [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "getEngineVersion1")] + private static extern int getEngineVersion1(); + [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "CreateMosesSystem")] + private static extern IntPtr CreateMosesSystem(string s); + [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", CallingConvention = CallingConvention.StdCall, EntryPoint = "GetMosesSystem")] + private static extern int GetMosesSystem(string s,ref IntPtr system); + [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "MosesTranslate")] + private static extern int MosesTranslate(IntPtr model,int id,string input,StringBuilder output,int output_len); + [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "ReleaseSystem")] + private static extern void ReleaseSystem(IntPtr model); + static void Main(string[] args) + { + string a = "D:/moses-mstranslator/test_sentence_with_candidates/moses_mspt.ini"; + string cand = "फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961"; + IntPtr system = new IntPtr(0); + int v = GetMosesSystem(a,ref system); + StringBuilder output = new StringBuilder(); + int error_code = MosesTranslate(system,1234678,cand,output,50); + Console.WriteLine(output); + + } + } +} + diff --git a/contrib/other-builds/moses2/moses2.vcxproj.filters b/contrib/other-builds/moses2/moses2.vcxproj.filters index 70a76ce23..83f43130c 100644 --- a/contrib/other-builds/moses2/moses2.vcxproj.filters +++ b/contrib/other-builds/moses2/moses2.vcxproj.filters @@ -67,508 +67,516 @@ {8f10d9c1-66d4-4490-b310-d3f4973bad29} + + {8f2f68ba-2b5e-4d2c-92a7-6ece71df74ba} + - - Source Files\util - - - Source Files\util - - - Source Files\util + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util\double-conversion + + Source Files\FF - - Source Files\util + + Source Files\FF - - Source Files\util + + Source Files\FF - - Source Files\util + + Source Files\FF - - Source Files\util + + Source Files\FF\OSM - - Source Files\FF + + Source Files\FF\OSM - - Source Files\FF + + Source Files\FF\OSM - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - - Source Files\FF + + Source Files\FF\LexicalReordering - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - + Source Files\legacy - - Source Files\parameters + + Source Files\Moses2LM - - Source Files\parameters + + Source Files\Moses2LM - + + Source Files\Moses2LM + + + Source Files\Moses2LM + + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - + Source Files\parameters - - Source Files\SCFG + + Source Files\parameters - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\SCFG + + Source Files\PhraseBased - - Source Files\server + + Source Files\PhraseBased\Normal - - Source Files\server + + Source Files\PhraseBased\Normal - - Source Files\server + + Source Files\PhraseBased\Normal - - Source Files + + Source Files\PhraseBased\CubePruningMiniStack - - Source Files + + Source Files\PhraseBased\CubePruningMiniStack - - Source Files + + Source Files\PhraseBased\CubePruningMiniStack - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\ProbingPT - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG - - Source Files + + Source Files\SCFG\nbest - - Source Files + + Source Files\SCFG\nbest - - Source Files\util + + Source Files\SCFG\nbest - - Source Files\TranslationModel\Memory + + Source Files\SCFG\nbest - + Source Files\TranslationModel - + Source Files\TranslationModel - + Source Files\TranslationModel - - Source Files\FF\LexicalReordering + + Source Files\TranslationModel - - Source Files\FF\LexicalReordering + + Source Files\TranslationModel\Memory - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\LexicalReordering + + Source Files\util\double-conversion - - Source Files\FF\OSM + + Source Files\util\double-conversion - - Source Files\FF\OSM + + Source Files\util\double-conversion - - Source Files\FF\OSM + + Source Files\util - + Source Files\util - + Source Files\util - + Source Files\util - + Source Files\util - + Source Files\util - + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files\util - - Source Files\PhraseBased + + Source Files - - Source Files\PhraseBased + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\lm + + Source Files - - Source Files\util + + Source Files - - Source Files\util + + Source Files - - Source Files\PhraseBased\Normal + + Source Files - - Source Files\PhraseBased\Normal + + Source Files - - Source Files\PhraseBased\Normal + + Source Files - - Source Files\PhraseBased\CubePruningMiniStack + + Source Files - - Source Files\PhraseBased\CubePruningMiniStack + + Source Files - - Source Files\PhraseBased\CubePruningMiniStack + + Source Files - - Source Files\Moses2LM + + Source Files\lm - - Source Files\Moses2LM + + Source Files\lm - - Source Files\Moses2LM + + Source Files\lm - - Source Files\Moses2LM + + Source Files\lm - - Source Files\SCFG\nbest + + Source Files\lm - - Source Files\SCFG\nbest + + Source Files\lm - - Source Files\SCFG\nbest + + Source Files\lm - - Source Files\SCFG\nbest + + Source Files\lm - - Source Files\TranslationModel + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files\lm - - Source Files\ProbingPT + + Source Files - - Source Files\FF + + Source Files\parameters - - Source Files\FF + + Source Files\TranslationModel\Dynamic + + + Source Files + + \ No newline at end of file diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp new file mode 100644 index 000000000..cb8def6c0 --- /dev/null +++ b/moses2/DLLEntryApi.cpp @@ -0,0 +1,39 @@ +#include "Moses2Wrapper.h" +using namespace std; +using namespace Moses2; + +extern "C" __declspec(dllexport) Moses2::Moses2Wrapper * __stdcall CreateMosesSystem(const char* filePath) { + return new Moses2::Moses2Wrapper(filePath); +} + +extern "C" __declspec(dllexport) int __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { + *pObject = new Moses2::Moses2Wrapper(filePath); + return 1; +} + +extern "C" __declspec(dllexport) int __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char * output, int strlen) { + if (pObject != NULL) + { + std::string tr = pObject->Translate(input, id); + std::copy(tr.begin(), tr.end(), output); + output[std::min(strlen - 1, (int)tr.size())] = 0; + return 1; + } + else { + return 0; + } +} +extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapper * pObject) { + if (pObject != NULL) + { + delete pObject; + pObject = NULL; + return 1; + } + else { + return 0; + } +} +extern "C" __declspec(dllexport) string __stdcall GetEngineVersion() { + return "1.0"; +} \ No newline at end of file diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index f919457d5..8fa4a5156 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -1,4 +1,4 @@ -#include "Moses2Wrapper.h" +#include "Moses2Wrapper.h" #include "System.h" #include "legacy/Parameter.h" #include "TranslationTask.h" @@ -12,7 +12,13 @@ namespace Moses2 { std::string Moses2Wrapper::Translate(const std::string &input , long id) { TranslationTask task(*m_system, input, id); std::string translation = task.ReturnTranslation(); - //delete translation; return translation; } + Moses2Wrapper* Moses2Wrapper::getInstance(const std::string& filePath) { + Moses2Wrapper *instance = new Moses2Wrapper(filePath); + return instance; + } + Moses2Wrapper::~Moses2Wrapper() { + cout << "Destructor is called "; + } } \ No newline at end of file diff --git a/moses2/Moses2Wrapper.h b/moses2/Moses2Wrapper.h index f491e1078..21aa19388 100644 --- a/moses2/Moses2Wrapper.h +++ b/moses2/Moses2Wrapper.h @@ -3,14 +3,18 @@ namespace Moses2 { class Parameter; class System; - class Moses2Wrapper + class Moses2Wrapper { Parameter *m_param; System *m_system; public: Moses2Wrapper(const std::string &filePath); + ~Moses2Wrapper(); std::string Translate(const std::string &input, long id); + Moses2Wrapper* getInstance(const std::string& filePath); + int getEngineVersion(); + }; } \ No newline at end of file -- cgit v1.2.3 From affcd000bd05392cf4a895dafababbd6052a2bdd Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 27 Oct 2020 11:50:02 -0700 Subject: dos2unix --- moses2/DLLEntryApi.cpp | 2 +- moses2/Moses2Wrapper.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index cb8def6c0..c9280b690 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -1,4 +1,4 @@ -#include "Moses2Wrapper.h" +#include "Moses2Wrapper.h" using namespace std; using namespace Moses2; diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index 8fa4a5156..833f18749 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -1,4 +1,4 @@ -#include "Moses2Wrapper.h" +#include "Moses2Wrapper.h" #include "System.h" #include "legacy/Parameter.h" #include "TranslationTask.h" -- cgit v1.2.3 From f40a77572333cbdb4aa18d70c2e8bdf74bba31e6 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Wed, 4 Nov 2020 16:32:48 +0530 Subject: changes to release memory --- moses2/DLLEntryApi.cpp | 11 ++++++----- moses2/Moses2Wrapper.cpp | 9 +++------ moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp | 14 +++++++------- moses2/TranslationModel/Dynamic/DynamicPhraseTable.h | 2 +- moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h | 4 +++- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index cb8def6c0..e6c785bb4 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -1,4 +1,5 @@ #include "Moses2Wrapper.h" +#include using namespace std; using namespace Moses2; @@ -11,7 +12,7 @@ extern "C" __declspec(dllexport) int __stdcall GetMosesSystem(const char* filePa return 1; } -extern "C" __declspec(dllexport) int __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char * output, int strlen) { +extern "C" __declspec(dllexport) int __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { if (pObject != NULL) { std::string tr = pObject->Translate(input, id); @@ -23,11 +24,11 @@ extern "C" __declspec(dllexport) int __stdcall MosesTranslate(Moses2::Moses2Wrap return 0; } } -extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapper * pObject) { - if (pObject != NULL) +extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapper ** pObject) { + if (*pObject != NULL) { - delete pObject; - pObject = NULL; + delete *pObject; + *pObject = NULL; return 1; } else { diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index 8fa4a5156..01f1b6e72 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -14,11 +14,8 @@ namespace Moses2 { std::string translation = task.ReturnTranslation(); return translation; } - Moses2Wrapper* Moses2Wrapper::getInstance(const std::string& filePath) { - Moses2Wrapper *instance = new Moses2Wrapper(filePath); - return instance; - } Moses2Wrapper::~Moses2Wrapper() { - cout << "Destructor is called "; - } + delete m_param; + delete m_system; + } } \ No newline at end of file diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp index 0a74fcaa3..a37f5b778 100644 --- a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp @@ -36,7 +36,7 @@ using namespace std; namespace Moses2 { -thread_local DynamicPhraseTable::PBNODE * DynamicPhraseTable::m_rootPb; +thread_local DynamicPhraseTable::PBNODE DynamicPhraseTable::m_rootPb; //////////////////////////////////////////////////////////////////////// @@ -48,7 +48,7 @@ DynamicPhraseTable::DynamicPhraseTable(size_t startInd, const std::string &line) DynamicPhraseTable::~DynamicPhraseTable() { - delete m_rootPb; + //delete m_rootPb; } void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) @@ -60,7 +60,7 @@ void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseT MemPool tmpSourcePool; if (system.isPb) { - m_rootPb = new PBNODE(); + //m_rootPb = new PBNODE(); } else { abort(); //cerr << "m_rootSCFG=" << m_rootSCFG << endl; @@ -104,7 +104,7 @@ void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseT system.featureFunctions.EvaluateInIsolation(pool, system, *source, *target); //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl; - m_rootPb->AddRule(m_input, *source, target); + m_rootPb.AddRule(m_input, *source, target); //cerr << "target=" << target->Debug(system) << endl; } else { @@ -113,7 +113,7 @@ void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseT } if (system.isPb) { - m_rootPb->SortAndPrune(m_tableLimit, pool, system); + m_rootPb.SortAndPrune(m_tableLimit, pool, system); //cerr << "root=" << &m_rootPb << endl; } else { abort(); @@ -139,12 +139,12 @@ TargetPhrases* DynamicPhraseTable::Lookup(const Manager &mgr, MemPool &pool, InputPath &inputPath) const { const SubPhrase &phrase = inputPath.subPhrase; - TargetPhrases *tps = m_rootPb->Find(m_input, phrase); + TargetPhrases *tps = m_rootPb.Find(m_input, phrase); return tps; } void DynamicPhraseTable::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const { - delete m_rootPb; + m_rootPb.CleanNode(); //TODO : clean this } void DynamicPhraseTable::InitActiveChart( diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h index 7fd467700..99588eda8 100644 --- a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.h @@ -69,7 +69,7 @@ public: virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const; protected: - thread_local static PBNODE *m_rootPb; + thread_local static PBNODE m_rootPb; void LookupGivenNode( MemPool &pool, diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h b/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h index b10e4e161..56eb181cb 100644 --- a/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTableNode.h @@ -88,7 +88,9 @@ public: delete m_unsortedTPS; } } - + void CleanNode() { + m_children.clear(); + } const Children &GetChildren() const { return m_children; } -- cgit v1.2.3 From bfc0bc57a34f64ae9a83a2ba181def04deb56ff4 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Tue, 10 Nov 2020 00:21:14 +0530 Subject: remaining clean up changes --- moses2/DLLEntryApi.cpp | 24 ++++++++++++---------- moses2/Moses2Wrapper.cpp | 3 +-- .../Dynamic/DynamicPhraseTable.cpp | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index e6c785bb4..b559b62e5 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -1,27 +1,29 @@ #include "Moses2Wrapper.h" #include +#include using namespace std; using namespace Moses2; -extern "C" __declspec(dllexport) Moses2::Moses2Wrapper * __stdcall CreateMosesSystem(const char* filePath) { - return new Moses2::Moses2Wrapper(filePath); -} - -extern "C" __declspec(dllexport) int __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { +extern "C" __declspec(dllexport) HRESULT __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { + if (*pObject == NULL) { *pObject = new Moses2::Moses2Wrapper(filePath); - return 1; + return S_OK; + } + else { + return E_FAIL; + } } -extern "C" __declspec(dllexport) int __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { +extern "C" __declspec(dllexport) HRESULT __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { if (pObject != NULL) { std::string tr = pObject->Translate(input, id); std::copy(tr.begin(), tr.end(), output); output[std::min(strlen - 1, (int)tr.size())] = 0; - return 1; + return S_OK; } else { - return 0; + return E_FAIL; } } extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapper ** pObject) { @@ -29,10 +31,10 @@ extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapp { delete *pObject; *pObject = NULL; - return 1; + return S_OK; } else { - return 0; + return E_FAIL; } } extern "C" __declspec(dllexport) string __stdcall GetEngineVersion() { diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index a6897037e..343ddfa3b 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -11,8 +11,7 @@ namespace Moses2 { } std::string Moses2Wrapper::Translate(const std::string &input , long id) { TranslationTask task(*m_system, input, id); - std::string translation = task.ReturnTranslation(); - return translation; + return task.ReturnTranslation(); } Moses2Wrapper::~Moses2Wrapper() { delete m_param; diff --git a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp index a37f5b778..a7edd3f84 100644 --- a/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp +++ b/moses2/TranslationModel/Dynamic/DynamicPhraseTable.cpp @@ -48,7 +48,7 @@ DynamicPhraseTable::DynamicPhraseTable(size_t startInd, const std::string &line) DynamicPhraseTable::~DynamicPhraseTable() { - //delete m_rootPb; + m_rootPb.CleanNode(); } void DynamicPhraseTable::CreatePTForInput(const ManagerBase &mgr, string phraseTableString) -- cgit v1.2.3 From b46da0dc507fd8bb34357647f7eae9a9cd7f49ad Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Mon, 23 Nov 2020 17:20:33 +0530 Subject: appending relative path to lm at runtime --- moses2/Moses2Wrapper.cpp | 29 +++++++++++++++++++++++++++++ moses2/Moses2Wrapper.h | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index 343ddfa3b..c699bfd5e 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -4,9 +4,38 @@ #include "TranslationTask.h" using namespace std; namespace Moses2 { + //summary :: need to update the LM path at runtime with complete artifact path. + void Moses2Wrapper::UpdateLMPath(const std::string& filePath) { + auto file = filePath.substr(filePath.find_last_of("\\") + 1); + auto path = filePath.substr(0, filePath.find_last_of("\\")); + auto a = m_param->GetParam("feature"); + std::vector feature; + for (int i = 0; i < a->size(); i++) { + auto abc = Tokenize(a->at(i)); + if (*abc.begin() == "KENLM") { + string s = ""; + for (int k = 0; k < abc.size(); k++) { + if (abc.at(k).find("path=") != string::npos) { + auto lm = abc.at(k).substr(abc.at(k).find_last_of("=") + 1); + s = s + "path=" + path + "\\" + lm + " "; + } + else { + s = s + abc.at(k) + " "; + } + } + feature.push_back(s.erase(s.find_last_not_of(" \n\r\t") + 1)); + } + else { + feature.push_back(a->at(i)); + } + } + m_param->OverwriteParam("feature", feature); + } + Moses2Wrapper::Moses2Wrapper(const std::string &filePath) { m_param = new Parameter(); m_param->LoadParam(filePath); + UpdateLMPath(filePath); m_system = new System(*m_param); } std::string Moses2Wrapper::Translate(const std::string &input , long id) { diff --git a/moses2/Moses2Wrapper.h b/moses2/Moses2Wrapper.h index 21aa19388..1a815e38f 100644 --- a/moses2/Moses2Wrapper.h +++ b/moses2/Moses2Wrapper.h @@ -12,7 +12,7 @@ namespace Moses2 { Moses2Wrapper(const std::string &filePath); ~Moses2Wrapper(); std::string Translate(const std::string &input, long id); - Moses2Wrapper* getInstance(const std::string& filePath); + void UpdateLMPath(const std::string &filePath); int getEngineVersion(); }; -- cgit v1.2.3 From c961624aa82700a642495015edc2897aefd19c79 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Wed, 25 Nov 2020 12:53:19 +0530 Subject: env separator --- moses2/Moses2Wrapper.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/moses2/Moses2Wrapper.cpp b/moses2/Moses2Wrapper.cpp index c699bfd5e..fb59d010f 100644 --- a/moses2/Moses2Wrapper.cpp +++ b/moses2/Moses2Wrapper.cpp @@ -6,8 +6,14 @@ using namespace std; namespace Moses2 { //summary :: need to update the LM path at runtime with complete artifact path. void Moses2Wrapper::UpdateLMPath(const std::string& filePath) { - auto file = filePath.substr(filePath.find_last_of("\\") + 1); - auto path = filePath.substr(0, filePath.find_last_of("\\")); + + char sep = '/'; + + #ifdef _WIN32 + sep = '\\'; + #endif + auto file = filePath.substr(filePath.find_last_of(sep) + 1); + auto path = filePath.substr(0, filePath.find_last_of(sep)); auto a = m_param->GetParam("feature"); std::vector feature; for (int i = 0; i < a->size(); i++) { @@ -17,7 +23,7 @@ namespace Moses2 { for (int k = 0; k < abc.size(); k++) { if (abc.at(k).find("path=") != string::npos) { auto lm = abc.at(k).substr(abc.at(k).find_last_of("=") + 1); - s = s + "path=" + path + "\\" + lm + " "; + s = s + "path=" + path + sep + lm + " "; } else { s = s + abc.at(k) + " "; -- cgit v1.2.3 From b8c5ad2bbccaf720244ddf7cd644300627dc965b Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Wed, 25 Nov 2020 21:22:47 +0530 Subject: relative path --- contrib/other-builds/dllImpCheck/Program.cs | 32 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/contrib/other-builds/dllImpCheck/Program.cs b/contrib/other-builds/dllImpCheck/Program.cs index 4fbd38696..fcdf88b95 100644 --- a/contrib/other-builds/dllImpCheck/Program.cs +++ b/contrib/other-builds/dllImpCheck/Program.cs @@ -11,27 +11,25 @@ namespace dllImpCheck { class Program { - - [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "getEngineVersion1")] - private static extern int getEngineVersion1(); - [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "CreateMosesSystem")] - private static extern IntPtr CreateMosesSystem(string s); - [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", CallingConvention = CallingConvention.StdCall, EntryPoint = "GetMosesSystem")] - private static extern int GetMosesSystem(string s,ref IntPtr system); - [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "MosesTranslate")] - private static extern int MosesTranslate(IntPtr model,int id,string input,StringBuilder output,int output_len); - [DllImport("D:/moses-mstranslator/contrib/other-builds/moses2/x64/Debug/moses2.dll", EntryPoint = "ReleaseSystem")] - private static extern void ReleaseSystem(IntPtr model); + + [DllImport("../../../moses2/x64/Debug/moses2.dll", CallingConvention = CallingConvention.StdCall, EntryPoint = "GetMosesSystem")] + private static extern int GetMosesSystem(string s, ref IntPtr system); + [DllImport("../../../moses2/x64/Debug/moses2.dll", CallingConvention = CallingConvention.StdCall, EntryPoint = "MosesTranslate")] + private static extern int MosesTranslate(IntPtr model, int id, string input, StringBuilder output, int output_len); + [DllImport("../../../moses2/x64/Debug/moses2.dll", CallingConvention = CallingConvention.StdCall, EntryPoint = "ReleaseSystem")] + private static extern int ReleaseSystem(ref IntPtr model); + static void Main(string[] args) { - string a = "D:/moses-mstranslator/test_sentence_with_candidates/moses_mspt.ini"; - string cand = "फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961"; - IntPtr system = new IntPtr(0); - int v = GetMosesSystem(a,ref system); + string config = "D:\\src\\moses-mstranslator\\test_sentence_with_candidates\\moses_mspt.ini"; + string cand = "aaj din main chaand nikla @@@ aaj ||| आज ||| 0.23034750595193718 $$$ aaj ||| अाज ||| 0.2036812076840512 $$$ aaj ||| एएजे ||| 0.1806033272478164 $$$ aaj ||| आज़ ||| 0.1550204531642581 $$$ din ||| दिन ||| 0.23292194982342979 $$$ din ||| दीन ||| 0.20844420805170855 $$$ din ||| दिं ||| 0.16399885041729953 $$$ din ||| डिन ||| 0.16171304188413235 $$$ chaand ||| चांद ||| 0.2374591084461087 $$$ chaand ||| चाँद ||| 0.217932729237165 $$$ chaand ||| चंद ||| 0.15435859487004985 $$$ chaand ||| चांड ||| 0.15279045900056767 $$$ nikla ||| निकला ||| 0.2727953350543125 $$$ nikla ||| निक्ला ||| 0.15350986400512082 $$$ nikla ||| नीकला ||| 0.1533410959941387 $$$ nikla ||| निकल़ा ||| 0.1475583698921154 $$$ main ||| मैं ||| 0.20812875019912347 $$$ main ||| में ||| 0.2042153102272697 $$$ main ||| मैन ||| 0.1933505532706236 $$$ main ||| मेन ||| 0.18617663610385968"; + IntPtr system = IntPtr.Zero; + int v = GetMosesSystem(config, ref system); StringBuilder output = new StringBuilder(); - int error_code = MosesTranslate(system,1234678,cand,output,50); + var ret = MosesTranslate(system, 1234678, cand, output, 50); Console.WriteLine(output); - + ReleaseSystem(ref system); + Console.ReadLine(); } } } -- cgit v1.2.3 From e9cc90e73ab26546906809d5af8ddd5c7e092add Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Fri, 18 Dec 2020 11:28:26 +0530 Subject: shared object generation --- moses2/DLLEntryApi.cpp | 32 +++++++++++++++++++++++++++----- moses2/Jamfile | 12 +++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index b559b62e5..155a6664c 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -1,10 +1,32 @@ #include "Moses2Wrapper.h" #include -#include +#ifdef WIN32 + #include +#endif // DEBUG + +#if defined(_MSC_VER) +// Microsoft + #define EXPORT __declspec(dllexport) + #define IMPORT __declspec(dllimport) +#elif defined(__GNUC__) +// GCC + #define HRESULT int + #define EXPORT __attribute__((visibility("default"))) + #define __stdcall + #define IMPORT + #define S_OK 0 + #define E_FAIL 1 +#else +// do nothing and hope for the best? + #define EXPORT + #define IMPORT + #pragma warning Unknown dynamic link import/export semantics. +#endif + using namespace std; using namespace Moses2; -extern "C" __declspec(dllexport) HRESULT __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { +extern "C" EXPORT HRESULT __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { if (*pObject == NULL) { *pObject = new Moses2::Moses2Wrapper(filePath); return S_OK; @@ -14,7 +36,7 @@ extern "C" __declspec(dllexport) HRESULT __stdcall GetMosesSystem(const char* fi } } -extern "C" __declspec(dllexport) HRESULT __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { +extern "C" EXPORT HRESULT __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { if (pObject != NULL) { std::string tr = pObject->Translate(input, id); @@ -26,7 +48,7 @@ extern "C" __declspec(dllexport) HRESULT __stdcall MosesTranslate(Moses2::Moses2 return E_FAIL; } } -extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapper ** pObject) { +extern "C" EXPORT int __stdcall ReleaseSystem(Moses2::Moses2Wrapper ** pObject) { if (*pObject != NULL) { delete *pObject; @@ -37,6 +59,6 @@ extern "C" __declspec(dllexport) int __stdcall ReleaseSystem(Moses2::Moses2Wrapp return E_FAIL; } } -extern "C" __declspec(dllexport) string __stdcall GetEngineVersion() { +extern "C" EXPORT string __stdcall GetEngineVersion() { return "1.0"; } \ No newline at end of file diff --git a/moses2/Jamfile b/moses2/Jamfile index 9c1a6840b..ed7f4cc97 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -52,7 +52,7 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors TypeDef.cpp Vector.cpp Weights.cpp - Word.cpp + Word.cpp FF/Distortion.cpp FF/FeatureFunction.cpp FF/FeatureFunctions.cpp @@ -188,9 +188,15 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors : $(includes) ; - +lib moses2so : + Moses2Wrapper.cpp + DLLEntryApi.cpp + moses2_lib + : shared + $(includes) + ; exe moses2 : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm ; echo "Building Moses2" ; -alias programs : moses2 ; +alias programs : moses2 moses2so ; -- cgit v1.2.3 From 1631aa14d15068bfb8d028371a51c15503220be3 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Tue, 12 Jan 2021 22:48:42 +0530 Subject: api signature update --- moses2/DLLEntryApi.cpp | 80 ++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index 155a6664c..e03ff9346 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -1,64 +1,74 @@ #include "Moses2Wrapper.h" #include -#ifdef WIN32 - #include -#endif // DEBUG +#include + + +// Generic helper definitions for shared library support +#if defined _WIN32 +#define IMPORT __declspec(dllimport) +#define EXPORT __declspec(dllexport) +#else // !(defined _WIN32 || defined __CYGWIN__) -- i.e., not Windows +#define __stdcall +#if __GNUC__ >= 4 +#define IMPORT __attribute__ ((visibility ("default"))) +#define EXPORT __attribute__ ((visibility ("default"))) +#else // __GNUC__ < 4, which does not support the __attribute__ tag +#define IMPORT +#define EXPORT +#endif // __GNUC__ >= 4 +#endif -#if defined(_MSC_VER) -// Microsoft - #define EXPORT __declspec(dllexport) - #define IMPORT __declspec(dllimport) -#elif defined(__GNUC__) -// GCC - #define HRESULT int - #define EXPORT __attribute__((visibility("default"))) - #define __stdcall - #define IMPORT - #define S_OK 0 - #define E_FAIL 1 -#else -// do nothing and hope for the best? - #define EXPORT - #define IMPORT - #pragma warning Unknown dynamic link import/export semantics. -#endif using namespace std; using namespace Moses2; -extern "C" EXPORT HRESULT __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper ** pObject) { +extern "C" EXPORT MosesApiErrorCode __stdcall GetMosesSystem(const char* filePath, Moses2::Moses2Wrapper * *pObject) { + if (*pObject == NULL) { *pObject = new Moses2::Moses2Wrapper(filePath); - return S_OK; + return MS_API_OK; } else { - return E_FAIL; + return MS_API_E_FAILURE; } } -extern "C" EXPORT HRESULT __stdcall MosesTranslate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char* output, int strlen) { +extern "C" EXPORT MosesApiErrorCode __stdcall Translate(Moses2::Moses2Wrapper * pObject, long id, const char* input, char** output) { if (pObject != NULL) { std::string tr = pObject->Translate(input, id); - std::copy(tr.begin(), tr.end(), output); - output[std::min(strlen - 1, (int)tr.size())] = 0; - return S_OK; + *output = Moses2Wrapper::CopyString(tr.c_str()); + return MS_API_OK; + } + else { + return MS_API_E_FAILURE; + } +} + +extern "C" EXPORT MosesApiErrorCode __stdcall FreeMemory(char* output) { + if (output != nullptr) { + Moses2Wrapper::Free(output); + return MS_API_OK; } else { - return E_FAIL; + return MS_API_E_FAILURE; } } -extern "C" EXPORT int __stdcall ReleaseSystem(Moses2::Moses2Wrapper ** pObject) { + +extern "C" EXPORT MosesApiErrorCode __stdcall ReleaseSystem(Moses2::Moses2Wrapper * *pObject) { if (*pObject != NULL) { - delete *pObject; + delete* pObject; *pObject = NULL; - return S_OK; + return MS_API_OK; } else { - return E_FAIL; + return MS_API_E_FAILURE; } } -extern "C" EXPORT string __stdcall GetEngineVersion() { - return "1.0"; + +extern "C" EXPORT MosesApiErrorCode __stdcall EngineVersion() { + //std::cout << "windows build on v1142/ msvc 14.27.29110"<< std::endl; + std::cout << "0.0.1" << std::endl; + return MS_API_OK; } \ No newline at end of file -- cgit v1.2.3 From 64b356c79da2dfd002a0cf3b3e808483556f3023 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Tue, 12 Jan 2021 22:51:31 +0530 Subject: enum header --- moses2/Moses2Wrapper.h | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/moses2/Moses2Wrapper.h b/moses2/Moses2Wrapper.h index 1a815e38f..c758ef2f3 100644 --- a/moses2/Moses2Wrapper.h +++ b/moses2/Moses2Wrapper.h @@ -1,20 +1,39 @@ #pragma once #include -namespace Moses2 { +#include +namespace Moses2 { class Parameter; class System; + extern "C" { + enum MosesApiErrorCode { + MS_API_OK, + MS_API_E_FAILURE, + MS_API_E_INPUT, + MS_API_E_TIMEOUT + }; + } class Moses2Wrapper { - Parameter *m_param; - System *m_system; + Parameter* m_param; + System* m_system; public: - Moses2Wrapper(const std::string &filePath); + Moses2Wrapper(const std::string& filePath); ~Moses2Wrapper(); - std::string Translate(const std::string &input, long id); - void UpdateLMPath(const std::string &filePath); + std::string Translate(const std::string& input, long id); + void UpdateLMPath(const std::string& filePath); int getEngineVersion(); + static char* CopyString(const char* str) { + int32_t size = (int32_t)strlen(str); + char* obj = (char*)malloc(size + 1); + memcpy(obj, str, size); + obj[size] = '\0'; + return obj; + } + static void Free(void* ptr) { + free(ptr); + } }; } \ No newline at end of file -- cgit v1.2.3 From bf2c915696f34ca2c1272931d2103a2a20e58a73 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Wed, 13 Jan 2021 10:06:54 +0530 Subject: jamfile --- moses2/Jamfile | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/moses2/Jamfile b/moses2/Jamfile index ed7f4cc97..43970a597 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -188,15 +188,8 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors : $(includes) ; -lib moses2so : - Moses2Wrapper.cpp - DLLEntryApi.cpp - moses2_lib - : shared - $(includes) - ; -exe moses2 : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm ; - +#need to figure out this +lib moses2 : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm : shared ; +#exe moses2 : moses2 ; echo "Building Moses2" ; -alias programs : moses2 moses2so ; - +alias programs : moses2 ; \ No newline at end of file -- cgit v1.2.3 From b73a685e915e0884553b91db0773b1269db40c00 Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Thu, 4 Feb 2021 11:23:27 +0530 Subject: typo and jamfile fixes --- moses2/DLLEntryApi.cpp | 2 +- moses2/Jamfile | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/moses2/DLLEntryApi.cpp b/moses2/DLLEntryApi.cpp index e03ff9346..060bea678 100644 --- a/moses2/DLLEntryApi.cpp +++ b/moses2/DLLEntryApi.cpp @@ -55,7 +55,7 @@ extern "C" EXPORT MosesApiErrorCode __stdcall FreeMemory(char* output) { } } -extern "C" EXPORT MosesApiErrorCode __stdcall ReleaseSystem(Moses2::Moses2Wrapper * *pObject) { +extern "C" EXPORT MosesApiErrorCode __stdcall ReleaseSystem(Moses2::Moses2Wrapper **pObject) { if (*pObject != NULL) { delete* pObject; diff --git a/moses2/Jamfile b/moses2/Jamfile index 43970a597..39a0267fd 100644 --- a/moses2/Jamfile +++ b/moses2/Jamfile @@ -181,7 +181,8 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors SCFG/nbest/NBest.cpp SCFG/nbest/NBests.cpp SCFG/nbest/NBestColl.cpp - + Moses2Wrapper.cpp + DLLEntryApi.cpp deps cmph mserver2 @@ -189,7 +190,7 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors $(includes) ; #need to figure out this -lib moses2 : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm : shared ; -#exe moses2 : moses2 ; +lib moses2decoder : Main.cpp moses2_lib ../probingpt//probingpt ../util//kenutil ../lm//kenlm ; +exe moses2 : moses2decoder ; echo "Building Moses2" ; -alias programs : moses2 ; \ No newline at end of file +alias programs : moses2 moses2decoder ; \ No newline at end of file -- cgit v1.2.3 From b1db9be80a1fbe895940d77074ed79a2dc2b1c3c Mon Sep 17 00:00:00 2001 From: siddharth jain Date: Thu, 4 Feb 2021 11:36:42 +0530 Subject: cleanup --- contrib/other-builds/moses2/moses2.vcxproj | 434 +++++++++++++---------------- moses2/FF/FeatureFunctions.cpp | 1 - 2 files changed, 197 insertions(+), 238 deletions(-) diff --git a/contrib/other-builds/moses2/moses2.vcxproj b/contrib/other-builds/moses2/moses2.vcxproj index cad2aa33e..a6412265f 100644 --- a/contrib/other-builds/moses2/moses2.vcxproj +++ b/contrib/other-builds/moses2/moses2.vcxproj @@ -1,18 +1,10 @@  - - Debug - Win32 - Debug x64 - - Release - Win32 - Release x64 @@ -22,31 +14,23 @@ {B4304E97-D37F-4022-BD03-841A4FAEE398} Win32Proj moses2 + 10.0.18362.0 + + - - Application - true - v140 - Unicode - - Application + DynamicLibrary true - v120 - Unicode - - - Application - false - v140 - true + v142 Unicode + false + false - Application + DynamicLibrary false - v120 + v142 true Unicode @@ -66,32 +50,14 @@ - - true - true - - - false + $(ProjectName)decoder false + $(ProjectName)decoder - - - - - Level3 - Disabled - WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - $(SolutionDir)\..\..\..\;$(SolutionDir)\..\..\..\..\xmlrpc-c\include\;$(SolutionDir)\..\..\..\..\boost_1_55_0\;%(AdditionalIncludeDirectories) - - - Console - true - - @@ -99,34 +65,26 @@ Level3 Disabled NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - $(SolutionDir)\..\..\..\..\cmph-2.0\src;$(SolutionDir)\..\..\..\..\zlib-1.2.8;$(SolutionDir)\..\..\..\..\xmlrpc-c\include\;$(SolutionDir)\..\..\..\..\mman-win32\;$(SolutionDir)\..\..\..\..\boost_1_56_0\;$(SolutionDir)\..\..\..\moses2\;$(SolutionDir)\..\..\..\;%(AdditionalIncludeDirectories) - MultiThreadedDLL + D:\src\mtmain\private\mt\zlib\src;D:\src\moses-mstranslator;D:\src\moses-mstranslator\moses2;D:\boost_1_74_0;%(AdditionalIncludeDirectories) + MultiThreadedDebugDLL $(IntDir)\%(RelativeDir)\%(Filename).obj false false + Default + + + ProgramDatabase + false + + Console true - $(SolutionDir)\..\..\..\..\xmlrpc-c\bin\Release-Static-x64;$(SolutionDir)\..\..\..\..\zlib-1.2.8\contrib\vstudio\vc11\x64\ZlibDllRelease;$(SolutionDir)\..\..\..\..\mman-win32\x64\Release;$(SolutionDir)\..\..\..\..\cmph-2.0\visualstudio2013\x64\Release;$(SolutionDir)\..\..\..\..\boost_1_56_0\lib64-msvc-12.0\ - libxmlrpc.lib;libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;zlibwapi.lib;mman.lib;cmph.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - - - Console - true - true - true + D:\boost_1_74_0\lib64-msvc-14.2;D:\src\mtmain\private\mt\zlib\src\dll\bin\x64\Debug + %(AdditionalDependencies) + LinkVerbose + false @@ -138,188 +96,190 @@ true true NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - $(SolutionDir)\..\..\..\..\cmph-2.0\src;$(SolutionDir)\..\..\..\..\zlib-1.2.8;$(SolutionDir)\..\..\..\..\xmlrpc-c\include\;$(SolutionDir)\..\..\..\..\mman-win32\;$(SolutionDir)\..\..\..\..\boost_1_56_0\;$(SolutionDir)\..\..\..\moses2\;$(SolutionDir)\..\..\..\;%(AdditionalIncludeDirectories) - MultiThreadedDLL + D:\src\mtmain\private\mt\zlib\src;D:\boost_1_74_0\;D:\src\moses-mstranslator;D:\src\moses-mstranslator\moses2;%(AdditionalIncludeDirectories) + MultiThreaded $(IntDir)\%(RelativeDir)\%(Filename).obj false + Default Console true true true - $(SolutionDir)\..\..\..\..\xmlrpc-c\bin\Release-Static-x64;$(SolutionDir)\..\..\..\..\zlib-1.2.8\contrib\vstudio\vc11\x64\ZlibDllRelease;$(SolutionDir)\..\..\..\..\mman-win32\x64\Release;$(SolutionDir)\..\..\..\..\cmph-2.0\visualstudio2013\x64\Release;$(SolutionDir)\..\..\..\..\boost_1_56_0\lib64-msvc-12.0\ - libxmlrpc.lib;libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;zlibwapi.lib;mman.lib;cmph.lib;%(AdditionalDependencies) + D:\boost_1_74_0\lib64-msvc-14.2;D:\src\mtmain\private\mt\zlib\src\dll\bin\x64\Debug + %(AdditionalDependencies) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp index c11a2aadf..3ea75b21d 100644 --- a/moses2/FF/FeatureFunctions.cpp +++ b/moses2/FF/FeatureFunctions.cpp @@ -67,7 +67,6 @@ void FeatureFunctions::Create() UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section"); BOOST_FOREACH(const std::string &line, *ffParams) { - //cerr << "line=" << line << endl; FeatureFunction *ff = Create(line); m_featureFunctions.push_back(ff); -- cgit v1.2.3