Add more error debug message to Operator::Run (#20793)

* add more err msg, test=develop

* add more unittests, test=develop

paddle/fluid/framework/CMakeLists.txt

@@ -127,6 +127,7 @@ cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope gl
     shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack)
 
 cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context)
+cc_test(operator_exception_test SRCS operator_exception_test.cc DEPS operator op_registry device_context)
 
 cc_library(version SRCS version.cc)
 cc_test(version_test SRCS version_test.cc DEPS version)

paddle/fluid/framework/op_call_stack.cc

@@ -26,17 +26,22 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
   if (attrs.count("sub_block") != 0) {
     return;
   }
-  auto &callstack = boost::get<std::vector<std::string>>(
-      attrs.at(OpProtoAndCheckerMaker::OpCreationCallstackAttrName()));
+
+  const std::vector<std::string> *callstack = nullptr;
+  auto iter = attrs.find(OpProtoAndCheckerMaker::OpCreationCallstackAttrName());
+  if (iter != attrs.end()) {
+    callstack = &boost::get<std::vector<std::string>>(iter->second);
+    if (callstack->empty()) callstack = nullptr;
+  }
 
   std::ostringstream sout;
   std::ostringstream sout_py_trace;
   // Step 1. Construct python call stack string
-  if (!callstack.empty()) {
+  if (callstack) {
     sout_py_trace << "\n------------------------------------------\n";
     sout_py_trace << "Python Call Stacks (More useful to users):";
     sout_py_trace << "\n------------------------------------------\n";
-    for (auto &line : callstack) {
+    for (auto &line : *callstack) {
       sout_py_trace << line;
     }
   }
@@ -55,7 +60,7 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
   sout << "C++ Call Stacks (More useful to developers):";
   sout << "\n--------------------------------------------\n";
   sout << exception->err_str_;
-  if (!callstack.empty()) {
+  if (callstack) {
     sout << "  [operator < " << type << " > error]";
   }
   exception->err_str_ = sout.str();

paddle/fluid/framework/operator.cc

@@ -183,7 +183,14 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
   } catch (platform::EnforceNotMet& exception) {
     framework::InsertCallStackInfo(Type(), Attrs(), &exception);
     throw std::move(exception);
+  } catch (platform::EOFException&) {
+    std::rethrow_exception(std::current_exception());
+  } catch (std::exception& ex) {
+    LOG(WARNING) << Type() << " raises an exception "
+                 << platform::demangle(typeid(ex).name()) << ", " << ex.what();
+    std::rethrow_exception(std::current_exception());
   } catch (...) {
+    LOG(WARNING) << Type() << " raises an unknown exception";
     std::rethrow_exception(std::current_exception());
   }
 }

paddle/fluid/framework/operator_exception_test.cc

@@ -0,0 +1,85 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/framework/operator.h"
+#include <exception>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/platform/place.h"
+
+namespace paddle {
+namespace framework {
+
+class ExceptionThrownOperator : public OperatorBase {
+ public:
+  using OperatorBase::OperatorBase;
+
+  template <typename T>
+  void SetException(T &&obj) {
+    exception_ = std::make_exception_ptr(std::forward<T>(obj));
+  }
+
+ protected:
+  void RunImpl(const Scope &, const platform::Place &) const override {
+    if (exception_) {
+      std::rethrow_exception(exception_);
+    }
+  }
+
+ private:
+  std::exception_ptr exception_{nullptr};
+};
+
+class StubException : public std::exception {
+ public:
+  const char *what() const noexcept override { return ""; }
+};
+
+template <typename T>
+bool ExceptionTestMain(T &&obj, bool set_exception) {
+  ExceptionThrownOperator op("", {}, {}, {});
+  if (set_exception) {
+    op.SetException(std::forward<T>(obj));
+  }
+  Scope scope;
+  try {
+    op.Run(scope, platform::CPUPlace());
+    return false;
+  } catch (T &) {
+    return true;
+  }
+}
+
+TEST(test_operator_exception, test_operator_exception) {
+  platform::EnforceNotMet enforce_not_met("", __FILE__, __LINE__);
+  ASSERT_TRUE(ExceptionTestMain(enforce_not_met, true));
+  ASSERT_FALSE(ExceptionTestMain(enforce_not_met, false));
+
+  platform::EOFException eof("", __FILE__, __LINE__);
+  ASSERT_TRUE(ExceptionTestMain(eof, true));
+  ASSERT_FALSE(ExceptionTestMain(eof, false));
+
+  StubException stub;
+  ASSERT_TRUE(ExceptionTestMain(stub, true));
+  ASSERT_FALSE(ExceptionTestMain(stub, false));
+
+  ASSERT_TRUE(ExceptionTestMain(1, true));
+  ASSERT_FALSE(ExceptionTestMain(1, false));
+}
+
+}  // namespace framework
+}  // namespace paddle