!8946 Add TraceManger around backend graph optimize

From: @irmo
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc

@@ -399,6 +399,7 @@ bool UbPatternFusion::ReplaceFusionOp(std::unordered_map<int64_t, BufferFusionIn
                                       session::KernelGraph *kernel_graph) const {
   MS_EXCEPTION_IF_NULL(buffer_fusion_infos);
   auto buffer_fusion_info = (*buffer_fusion_infos)[fusion_id];
+  TraceGuard guard(std::make_shared<TraceOpt>(buffer_fusion_info.anf_nodes[0]->debug_info()));
   auto buffer_fusion = CreateFusionOp(buffer_fusion_info.inputs_list, buffer_fusion_info.outputs_list,
                                       buffer_fusion_info.anf_nodes, kernel_graph);
   AnfAlgo::SetSelectKernelBuildInfo(buffer_fusion_info.kernel_build_info, buffer_fusion.get());

mindspore/ccsrc/backend/optimizer/ascend/format_type/convert_unsupported_transnode_to_aicpu.cc

@@ -47,7 +47,7 @@ const AnfNodePtr ConvertUnSupportNodeToAICPU::Process(const mindspore::FuncGraph
     AnfAlgo::SetNodeAttr(kAttrIsAICPUKernel, MakeValue(true), node);
   } else {
     MS_LOG(EXCEPTION) << " kernel " << kernel_builder_info->ToString() << "is not supported in AiCPU & AiCore : node ["
-                      << node->DebugString() << "]";
+                      << node->DebugString() << "]" << trace::DumpSourceLines(node);
   }
   return nullptr;
 }

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc

@@ -35,6 +35,7 @@ bool TransDataSplit::Run(const FuncGraphPtr &func_graph) {
     if (node != nullptr && node->isa<CNode>() && AnfAlgo::GetCNodeName(node) == kTransDataOpName) {
       CheckCNodeInputSize(node->cast<CNodePtr>(), kBackendTransDataInputNum);
       if (IsFormatInvaild(node)) {
+        TraceGuard guard(std::make_shared<TraceOpt>(node->debug_info()));
         changed = DoSplit(func_graph, node);
       }
     }

mindspore/ccsrc/backend/optimizer/common/node_pass.cc

@@ -40,6 +40,7 @@ bool NodePass::Run(const FuncGraphPtr &func_graph) {
       continue;
     }
     (void)seen_node.insert(node);
+    TraceGuard guard(std::make_shared<TraceOpt>(node->debug_info()));
     AnfNodePtr new_node = Run(func_graph, node);
     bool change = (new_node != nullptr);
     if (new_node != nullptr && new_node != node) {

mindspore/ccsrc/backend/optimizer/common/pass.h

@@ -20,6 +20,7 @@
 
 #include "ir/anf.h"
 #include "base/core_ops.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {

mindspore/ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc

@@ -99,6 +99,7 @@ bool CastAllFusion::Run(const FuncGraphPtr &graph) {
     for (size_t idx = 0; idx < cast_list.size(); ++idx) {
       inputs.push_back(AnfAlgo::GetInputNode(utils::cast<CNodePtr>(cast_list[idx]), 0));
     }
+    TraceGuard guard(std::make_shared<TraceOpt>(cast_list[0]->debug_info()));
     auto cast_all = graph->NewCNode(inputs);
     auto kernel_info = std::make_shared<device::KernelInfo>();
     MS_EXCEPTION_IF_NULL(kernel_info);

mindspore/ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc

@@ -104,6 +104,7 @@ bool CombineMomentumFusion::Run(const FuncGraphPtr &graph) {
         inputs.push_back(AnfAlgo::GetInputNode(utils::cast<CNodePtr>(mom), i));
       }
     }
+    TraceGuard guard(std::make_shared<TraceOpt>(momentums[0]->debug_info()));
     auto combine_mom = graph->NewCNode(inputs);
     auto kernel_info = std::make_shared<device::KernelInfo>();
     MS_EXCEPTION_IF_NULL(kernel_info);

mindspore/ccsrc/backend/optimizer/graph_kernel/arithmetic_simplify.cc

@@ -32,7 +32,9 @@ namespace opt {
 AnfNodePtr NewCNodeWithInfo(const AnfNodePtrList &inputs, const AnfNodePtr &ori_node) {
   auto func_graph = ori_node->func_graph();
   MS_EXCEPTION_IF_NULL(func_graph);
+  TraceManager::DebugTrace(std::make_shared<TraceOpt>(ori_node->debug_info()));
   auto new_cnode = func_graph->NewCNode(inputs);
+  TraceManager::EndTrace();
   new_cnode->set_abstract(ori_node->abstract());
   new_cnode->set_kernel_info(std::make_shared<device::KernelInfo>());
   if (func_graph->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL)) {
@@ -687,6 +689,7 @@ void InlineSubgraph(const CNodePtr &kernel_node, const FuncGraphPtr &sub_graph,
 
 CNodePtr AddIdentityToEmptyPath(const AnfNodePtr &node, const FuncGraphPtr &sub_graph) {
   if (node->isa<Parameter>() || node->isa<ValueNode>()) {
+    TraceGuard guard(std::make_shared<TraceOpt>(node->debug_info()));
     auto identity_node = sub_graph->NewCNode({NewValueNode(prim::kPrimIdentity), node});
     identity_node->set_abstract(node->abstract());
     sub_graph->AddNode(identity_node);

mindspore/ccsrc/backend/optimizer/graph_kernel/graph_kernel_splitter.cc

@@ -265,6 +265,7 @@ class AreaGraph {
   CNodePtr CreateMainCNode(const FuncGraphPtr &main_func_graph, const FuncGraphPtr &sub_func_graph,
                            const std::vector<CNodePtr> &main_cnodes,
                            const std::unordered_map<ParameterPtr, AnfNodePtr> &param_node_map) {
+    TraceGuard guard(std::make_shared<TraceOpt>(sub_func_graph->debug_info()));
     AnfNodePtrList main_cnode_inputs = {NewValueNode(sub_func_graph)};
     for (const auto &param : sub_func_graph->parameters()) {
       // assert the param exists.
@@ -276,6 +277,7 @@ class AreaGraph {
         auto idx = NewValueNode(idx_val);
         idx->set_abstract(std::make_shared<abstract::AbstractScalar>(idx_val));
         AnfNodePtrList getitem_inputs = {NewValueNode(prim::kPrimTupleGetItem), main_cnodes[input_area], idx};
+        TraceGuard g_sub(std::make_shared<TraceOpt>(main_cnodes[input_area]->debug_info()));
         auto getitem_node = main_func_graph->NewCNode(getitem_inputs);
         getitem_node->set_abstract(main_cnodes[input_area]->abstract());
         main_cnode_inputs.push_back(getitem_node);

mindspore/ccsrc/backend/optimizer/graph_kernel/shape_ops_splitter.cc

@@ -46,6 +46,7 @@ AnfNodePtr CloneCNode(const AnfNodePtr &anf_node) {
   MS_EXCEPTION_IF_NULL(kernel_graph);
   auto cnode = anf_node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(cnode);
+  TraceGuard guard(std::make_shared<TraceOpt>(cnode->debug_info()));
   CNodePtr node = kernel_graph->NewCNode(cnode->inputs());
   node->set_abstract(cnode->abstract());
   node->set_forward(cnode->forward().first, cnode->forward().second);

mindspore/ccsrc/backend/optimizer/pass/communication_op_fusion.cc

@@ -278,6 +278,7 @@ bool CommunicationOpFusion::Run(const FuncGraphPtr &func_graph) {
       continue;
     }
     auto first_node = it.second.communication_op_nodes[0];
+    TraceGuard guard(std::make_shared<TraceOpt>(first_node->debug_info()));
     if (AnfAlgo::HasNodeAttr(kAttrIndex, first_node) && AnfAlgo::GetNodeAttr<int64_t>(first_node, kAttrIndex) > 0) {
       std::stable_sort(it.second.communication_op_nodes.begin(), it.second.communication_op_nodes.end(),
                        [](const CNodePtr &a, const CNodePtr &b) {

mindspore/ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc

@@ -57,6 +57,7 @@ bool ReplaceNodeByProxy::Run(const FuncGraphPtr &func_graph) {
   std::vector<AnfNodePtr> node_list = TopoSort(func_graph->get_return());
   for (auto node : node_list) {
     if (node != nullptr && node->isa<CNode>() && AnfAlgo::GetCNodeName(node) == kEmbeddingLookupOpName) {
+      TraceGuard guard(std::make_shared<TraceOpt>(node->debug_info()));
       CNodePtr cnode = node->cast<CNodePtr>();
       auto prim = std::make_shared<Primitive>(kEmbeddingLookupProxyOpName);
       MS_EXCEPTION_IF_NULL(prim);

mindspore/ccsrc/backend/session/session_basic.cc

@@ -36,6 +36,7 @@
 #include "utils/utils.h"
 #include "debug/anf_ir_dump.h"
 #include "mindspore/core/base/base_ref_utils.h"
+#include "utils/trace_base.h"
 
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/worker.h"
@@ -992,8 +993,9 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
     }
     // Create cnode
     if (!CreateCNodeOfKernelGraph(node, graph.get())) {
-      DumpIR("contruct_kernel_graph_fail.ir", func_graph);
+      DumpIR("construct_kernel_graph_fail.ir", func_graph);
-      MS_LOG_EXCEPTION << "construct func graph " << func_graph->ToString() << "fail!";
+      MS_LOG(EXCEPTION) << "Construct func graph " << func_graph->ToString() << " failed."
+                        << trace::DumpSourceLines(node);
     }
   }