introduce log_tag option (#517)

17 files changed, 25 insertions, 16 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01f5b185..ae55ed43 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
 * '-' stands for stdin for inference tools (translate, lm, tag)
 * Optional source features per request (for domain control) with rest translation server
 * Display oov rate (source/target) in translate
+* Add `-log_tag` option to add tag in logs for better automatic processing
 
 ### Fixes and improvements
 
diff --git a/benchmark/generate_metadata.lua b/benchmark/generate_metadata.lua
index 10996c8a..ff7dfd5c 100644
--- a/benchmark/generate_metadata.lua
+++ b/benchmark/generate_metadata.lua
@@ -56,7 +56,7 @@ end
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   onmt.utils.Cuda.init(opt)
 
diff --git a/lm.lua b/lm.lua
index 1afb9f53..7298919f 100644
--- a/lm.lua
+++ b/lm.lua
@@ -38,7 +38,7 @@ cmd:option('-time', false, [[Measure average translation time.]])
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   onmt.utils.Cuda.init(opt)
 
   local lm = onmt.lm.LM.new(opt)
diff --git a/onmt/utils/Logger.lua b/onmt/utils/Logger.lua
index 65fde511..e1032bed 100644
--- a/onmt/utils/Logger.lua
+++ b/onmt/utils/Logger.lua
@@ -18,6 +18,10 @@ local options = {
     {
       enum = { 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'NONE' }
     }
+  },
+  {
+    '-log_tag', '',
+    [[Add specific tag to each log for better log processing in complex workflow.]]
   }
 }
 
@@ -39,7 +43,7 @@ Example:
     logger:shutDown()
 
 ]]
-function Logger:__init(logFile, disableLogs, logLevel)
+function Logger:__init(logFile, disableLogs, logLevel, logTag)
   logFile = logFile or ''
   self.jsonLog = logFile:sub(-5) == '.json'
   disableLogs = disableLogs or false
@@ -59,6 +63,7 @@ function Logger:__init(logFile, disableLogs, logLevel)
   else
     self.logFile = nil
   end
+  self.logTag = logTag
 end
 
 local function jsonize(msg)
@@ -82,6 +87,9 @@ function Logger:log(message, level)
     end
   else
     local msgFormatted = string.format('[%s %s] %s', timeStamp, level, message)
+    if self.logTag and self.logTag ~= '' then
+      msgFormatted = self.logTag..' '..msgFormatted
+    end
     if (not self.mute) and self:_isVisible(level) then
       print (msgFormatted)
     end
diff --git a/preprocess.lua b/preprocess.lua
index 38b2602f..f0eb1926 100644
--- a/preprocess.lua
+++ b/preprocess.lua
@@ -61,7 +61,7 @@ local function main()
 
   torch.manualSeed(opt.seed)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   _G.hookManager = onmt.utils.HookManager.new(opt)
 
diff --git a/tag.lua b/tag.lua
index 4279839e..f50f487d 100644
--- a/tag.lua
+++ b/tag.lua
@@ -39,7 +39,7 @@ cmd:option('-time', false, [[Measure average translation time.]])
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   onmt.utils.Cuda.init(opt)
 
   local tagger = onmt.tagger.Tagger.new(opt)
diff --git a/tools/apply_embeddings.lua b/tools/apply_embeddings.lua
index 7a73f5e2..904f14f4 100644
--- a/tools/apply_embeddings.lua
+++ b/tools/apply_embeddings.lua
@@ -47,7 +47,7 @@ onmt.utils.Logger.declareOpts(cmd)
 local opt = cmd:parse(arg)
 
 local function main()
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   local embeddingWeights = torch.load(opt.embed_data)
 
diff --git a/tools/average_models.lua b/tools/average_models.lua
index de5a0c2f..380251ed 100644
--- a/tools/average_models.lua
+++ b/tools/average_models.lua
@@ -66,7 +66,7 @@ local function main()
     assert(path.exists(f), 'model \'' .. f .. '\' does not exist.')
   end
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   if not opt.force then
     assert(not path.exists(opt.output_model),
diff --git a/tools/build_vocab.lua b/tools/build_vocab.lua
index 2dbe3891..858f95cf 100644
--- a/tools/build_vocab.lua
+++ b/tools/build_vocab.lua
@@ -47,7 +47,7 @@ end
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   local vocab = onmt.data.Vocabulary.init('source', opt.data, '', opt.vocab_size, opt.words_min_frequency, '', isValid, opt.keep_frequency, opt.idx_files)
 
diff --git a/tools/embeddings.lua b/tools/embeddings.lua
index 61177f9c..4ebb67db 100644
--- a/tools/embeddings.lua
+++ b/tools/embeddings.lua
@@ -403,7 +403,7 @@ end
 
 
 local function main()
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   local timer = torch.Timer()
 
diff --git a/tools/learn_bpe.lua b/tools/learn_bpe.lua
index 5c447fc2..f1705cdb 100644
--- a/tools/learn_bpe.lua
+++ b/tools/learn_bpe.lua
@@ -292,7 +292,7 @@ end
 
 local function main()
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   _G.hookManager = HookManager.new(opt)
 
   local vocab = get_vocabulary()
diff --git a/tools/release_model.lua b/tools/release_model.lua
index fc8e1cc6..e80f09eb 100644
--- a/tools/release_model.lua
+++ b/tools/release_model.lua
@@ -65,7 +65,7 @@ end
 local function main()
   assert(path.exists(opt.model), 'model \'' .. opt.model .. '\' does not exist.')
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
 
   if opt.output_model:len() == 0 then
     if opt.model:sub(-3) == '.t7' then
diff --git a/tools/rest_translation_server.lua b/tools/rest_translation_server.lua
index a53284c0..4dc31dcd 100644
--- a/tools/rest_translation_server.lua
+++ b/tools/rest_translation_server.lua
@@ -165,7 +165,7 @@ end
 
 local function main()
   -- load logger
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   onmt.utils.Cuda.init(opt)
 
   _G.hookManager = onmt.utils.HookManager.new(opt)
diff --git a/tools/score.lua b/tools/score.lua
index 5040a8c4..15856294 100644
--- a/tools/score.lua
+++ b/tools/score.lua
@@ -64,7 +64,7 @@ local function main()
     tok_options['mode'] = 'aggressive'
   end
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   _G.hookManager = require('onmt.utils.HookManager').new()
 
   -- read the references
diff --git a/tools/translation_server.lua b/tools/translation_server.lua
index e37d929a..ad449221 100644
--- a/tools/translation_server.lua
+++ b/tools/translation_server.lua
@@ -72,7 +72,7 @@ end
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   onmt.utils.Cuda.init(opt)
 
   _G.logger:info("Loading model")
diff --git a/train.lua b/train.lua
index a1dad9d5..b2caf671 100644
--- a/train.lua
+++ b/train.lua
@@ -293,7 +293,7 @@ local function main()
   torch.manualSeed(opt.seed)
 
   -- Initialize global context.
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   _G.crayon_logger = onmt.utils.CrayonLogger.new(opt)
   _G.profiler = onmt.utils.Profiler.new(false)
 
diff --git a/translate.lua b/translate.lua
index a1175fcf..34c7794d 100644
--- a/translate.lua
+++ b/translate.lua
@@ -71,7 +71,7 @@ end
 local function main()
   local opt = cmd:parse(arg)
 
-  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level)
+  _G.logger = onmt.utils.Logger.new(opt.log_file, opt.disable_logs, opt.log_level, opt.log_tag)
   _G.profiler = onmt.utils.Profiler.new()
   _G.hookManager = onmt.utils.HookManager.new(opt)