diff options
| -rw-r--r-- | CMakeLists.txt | 4 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 8 | ||||
| -rw-r--r-- | src/decoder/config.cpp | 18 |
4 files changed, 29 insertions, 11 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cb1b952..eab19e24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ LIST(APPEND CUDA_NVCC_FLAGS --default-stream per-thread; -std=c++11; -g; -O3; -a add_definitions(-DCUDA_API_PER_THREAD_DEFAULT_STREAM) SET(CUDA_PROPAGATE_HOST_FLAGS OFF) -include_directories(${amunn_SOURCE_DIR}) +include_directories(${amunmt_SOURCE_DIR}) find_package(CUDA REQUIRED) find_package(Boost COMPONENTS system filesystem program_options timer iostreams) @@ -53,5 +53,5 @@ if (LIBLZMA_FOUND) endif (LIBLZMA_FOUND) ### KenLM stuff - END ### -include_directories($amunn_SOURCE_DIR}/src) +include_directories($amunmt_SOURCE_DIR}/src) add_subdirectory(src) @@ -1,10 +1,10 @@ -# amuNN +# AmuNMT A C++ decoder for Neural Machine Translation (NMT) models trained with Theano-based scripts from Nematus (https://github.com/rsennrich/nematus) or DL4MT (https://github.com/nyu-dl/dl4mt-tutorial) -We aim at keeping compatibility with Nematus (at least as long as there is no training framework in amunNN), the continued compatbility with DL4MT will not be guaranteed. +We aim at keeping compatibility with Nematus (at least as long as there is no training framework in AmunNMT), the continued compatbility with DL4MT will not be guaranteed. ## Requirements: * CMake 3.5.1 (due to CUDA related bugs in earlier versions) @@ -34,7 +34,7 @@ On Ubuntu 16.04, you currently need g++4.9 to compile and cuda-7.5, this also re -DCMAKE_CXX_COMPILER=g++-4.9 -DCUDA_HOST_COMPILER=/usr/bin/g++-4.9 ## Vocabulary files -Vocabulary files (and all other config files) in amuNN are by default YAML files. amuNN also reads gzipped yml.gz files. +Vocabulary files (and all other config files) in AmuNMT are by default YAML files. AmuNMT also reads gzipped yml.gz files. * Vocabulary files from models trained with Nematus can be used directly as JSON is a proper subset of YAML. * Vocabularies for models trained with DL4MT (*.pkl extension) need to be converted to JSON/YAML with either of the two scripts below: @@ -43,9 +43,9 @@ python scripts/pkl2json.py vocab.en.pkl > vocab.json python scripts/pkl2yaml.py vocab.en.pkl > vocab.yml ``` -## Running amuNN +## Running AmuNMT - ./bin/amunn -c config.yml <<< "This is a test ." + ./bin/amun -c config.yml <<< "This is a test ." ## Configuration files diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d2deb379..28a934f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,19 +15,19 @@ add_library(librescorer OBJECT rescorer/nbest.cpp ) -add_library(libamunn OBJECT +add_library(libamun OBJECT decoder/config.cpp # decoder/kenlm.cpp ) cuda_add_executable( - amunn + amun decoder/decoder_main.cu decoder/god.cu decoder/sentence.cu mblas/matrix.cu dl4mt/gru.cu - $<TARGET_OBJECTS:libamunn> + $<TARGET_OBJECTS:libamun> $<TARGET_OBJECTS:libcommon> ) @@ -40,7 +40,7 @@ cuda_add_executable( $<TARGET_OBJECTS:libcommon> ) -foreach(exec amunn rescorer) +foreach(exec amun rescorer) target_link_libraries(${exec} ${EXT_LIBS} cuda) cuda_add_cublas_to_target(${exec}) set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") diff --git a/src/decoder/config.cpp b/src/decoder/config.cpp index 0842267c..321b7362 100644 --- a/src/decoder/config.cpp +++ b/src/decoder/config.cpp @@ -52,6 +52,16 @@ void ProcessPaths(YAML::Node& node, const boost::filesystem::path& configPath, b } } +void OverwriteModels(YAML::Node& config, std::vector<std::string>& modelPaths) { + //config["scorers"].clear(); + for(size_t i = 0; i < modelPaths.size(); ++i) { + std::stringstream name; + name << "F" << i; + config["scorers"][name.str()]["type"] = "Nematus"; + config["scorers"][name.str()]["path"] = modelPaths[i]; + } +} + void Validate(const YAML::Node& config) { UTIL_THROW_IF2(!config["scorers"] || config["scorers"].size() == 0, "No scorers given in config file"); @@ -157,10 +167,14 @@ void Config::AddOptions(size_t argc, char** argv) { "Output n-best list with n = beam-size") ; + std::vector<std::string> modelPaths; po::options_description configuration("Configuration meta options"); configuration.add_options() ("relative-paths", po::value<bool>()->zero_tokens()->default_value(false), "All paths are relative to the config file location") + ("model,m", po::value(&modelPaths)->multitoken(), + "Overwrite scorer section in config file with these models. " + "Assumes models of type Nematus and assigns model names F0, F1, ...") //("config-scorer", po::value<std::string>(), // "Overwrite scorer configuration with YAML string") //("config-weights", po::value<std::string>(), @@ -214,6 +228,10 @@ void Config::AddOptions(size_t argc, char** argv) { LoadWeights(config_, Get<std::string>("load-weights")); } + if(modelPaths.size()) { + OverwriteModels(config_, modelPaths); + } + if(Get<bool>("relative-paths")) ProcessPaths(config_, boost::filesystem::path{configPath}.parent_path(), false); Validate(config_); |