now catches SIGFPE with call stack

2 files changed, 33 insertions, 13 deletions

diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp
index e5515c7e..b60d1b5a 100755
--- a/src/command/marian_main.cpp
+++ b/src/command/marian_main.cpp
@@ -17,21 +17,32 @@
 #include "marian_conv.cpp"
 #undef main
 
+#include "3rd_party/ExceptionWithCallStack.h"
+
 int main(int argc, char** argv) {
   using namespace marian;
 
-  if(argc > 1 && argv[1][0] != '-') {
-    std::string cmd = argv[1];
-    argc--;
-    argv[1] = argv[0];
-    argv++;
-    if(cmd == "train")           return mainTrainer(argc, argv);
-    else if(cmd == "decode")     return mainDecoder(argc, argv);
-    // else if (cmd == "score")     return mainScorer(argc, argv);
-    else if (cmd == "vocab")     return mainVocab(argc, argv);
-    else if (cmd == "convert")   return mainConv(argc, argv);
-    std::cerr << "Command must be train, decode, score, vocab, or convert.";
+  try
+  {
+    if(argc > 1 && argv[1][0] != '-') {
+      std::string cmd = argv[1];
+      argc--;
+      argv[1] = argv[0];
+      argv++;
+      if(cmd == "train")           return mainTrainer(argc, argv);
+      else if(cmd == "decode")     return mainDecoder(argc, argv);
+      // else if (cmd == "score")     return mainScorer(argc, argv);
+      else if (cmd == "vocab")     return mainVocab(argc, argv);
+      else if (cmd == "convert")   return mainConv(argc, argv);
+      std::cerr << "Command must be train, decode, score, vocab, or convert." << std::endl;
+      exit(1);
+    } else
+      return mainTrainer(argc, argv);
+  }
+  catch (const std::exception& e)
+  {
+    std::cerr << "Command failed with exception: " << e.what() << std::endl
+              << ::Microsoft::MSR::CNTK::DebugUtil::GetCallStack(/*skipLevels=*/0/*2*/, /*makeFunctionNamesStandOut=*/true);
     exit(1);
-  } else
-    return mainTrainer(argc, argv);
+  }
 }
diff --git a/src/common/logging.cpp b/src/common/logging.cpp
index e01556ed..9acf07f3 100755
--- a/src/common/logging.cpp
+++ b/src/common/logging.cpp
@@ -108,6 +108,7 @@ void createLoggers(const marian::Config* options) {
 #ifdef __unix__
   // catch segfaults
   static struct sigaction prev_segfault_sigaction;
+  static struct sigaction prev_fperror_sigaction;
   struct sigaction sa = { 0 };
   sigemptyset(&sa.sa_mask);
   sa.sa_flags = SA_SIGINFO;
@@ -119,6 +120,14 @@ void createLoggers(const marian::Config* options) {
     raise(signal); // re-raise so we terminate mostly as usual
   };
   sigaction(SIGSEGV, &sa, &prev_segfault_sigaction);
+  sa.sa_sigaction = [&](int signal, siginfo_t *si, void *arg)
+  {
+      checkedLog("general", "critical", "Floating-point exception");
+      sigaction(signal, &prev_fperror_sigaction, NULL); // revert signal handler
+      marian::logCallStack(/*skipLevels=*/0/*2*/); // skip segfault_sigaction() and one level up in the kernel
+      raise(signal); // re-raise so we terminate mostly as usual
+  };
+  sigaction(SIGFPE, &sa, &prev_fperror_sigaction);
 #endif
 }