From e17dd84c0bc2023bba61bd18c0ff966694ff860e Mon Sep 17 00:00:00 2001 From: huanghui Date: Mon, 30 Nov 2020 10:47:07 +0800 Subject: [PATCH] add trace managager around backend opt --- .../optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc | 1 + .../format_type/convert_unsupported_transnode_to_aicpu.cc | 2 +- .../backend/optimizer/ascend/ir_fission/transdata_split.cc | 1 + mindspore/ccsrc/backend/optimizer/common/node_pass.cc | 1 + mindspore/ccsrc/backend/optimizer/common/pass.h | 1 + .../ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc | 1 + .../ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc | 1 + .../backend/optimizer/graph_kernel/arithmetic_simplify.cc | 3 +++ .../backend/optimizer/graph_kernel/graph_kernel_splitter.cc | 2 ++ .../backend/optimizer/graph_kernel/shape_ops_splitter.cc | 1 + .../ccsrc/backend/optimizer/pass/communication_op_fusion.cc | 1 + .../ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc | 1 + mindspore/ccsrc/backend/session/session_basic.cc | 6 ++++-- 13 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc index ce70d38ef1..91727eca32 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/buffer_fusion/ub_pattern_fusion.cc @@ -399,6 +399,7 @@ bool UbPatternFusion::ReplaceFusionOp(std::unordered_map(buffer_fusion_info.anf_nodes[0]->debug_info())); auto buffer_fusion = CreateFusionOp(buffer_fusion_info.inputs_list, buffer_fusion_info.outputs_list, buffer_fusion_info.anf_nodes, kernel_graph); AnfAlgo::SetSelectKernelBuildInfo(buffer_fusion_info.kernel_build_info, buffer_fusion.get()); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/format_type/convert_unsupported_transnode_to_aicpu.cc b/mindspore/ccsrc/backend/optimizer/ascend/format_type/convert_unsupported_transnode_to_aicpu.cc index 7b4e525c35..22955f547d 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/format_type/convert_unsupported_transnode_to_aicpu.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/format_type/convert_unsupported_transnode_to_aicpu.cc @@ -47,7 +47,7 @@ const AnfNodePtr ConvertUnSupportNodeToAICPU::Process(const mindspore::FuncGraph AnfAlgo::SetNodeAttr(kAttrIsAICPUKernel, MakeValue(true), node); } else { MS_LOG(EXCEPTION) << " kernel " << kernel_builder_info->ToString() << "is not supported in AiCPU & AiCore : node [" - << node->DebugString() << "]"; + << node->DebugString() << "]" << trace::DumpSourceLines(node); } return nullptr; } diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc index ef0fe087c7..94b74e847f 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc @@ -35,6 +35,7 @@ bool TransDataSplit::Run(const FuncGraphPtr &func_graph) { if (node != nullptr && node->isa() && AnfAlgo::GetCNodeName(node) == kTransDataOpName) { CheckCNodeInputSize(node->cast(), kBackendTransDataInputNum); if (IsFormatInvaild(node)) { + TraceGuard guard(std::make_shared(node->debug_info())); changed = DoSplit(func_graph, node); } } diff --git a/mindspore/ccsrc/backend/optimizer/common/node_pass.cc b/mindspore/ccsrc/backend/optimizer/common/node_pass.cc index 581361f4ae..58159f1359 100644 --- a/mindspore/ccsrc/backend/optimizer/common/node_pass.cc +++ b/mindspore/ccsrc/backend/optimizer/common/node_pass.cc @@ -40,6 +40,7 @@ bool NodePass::Run(const FuncGraphPtr &func_graph) { continue; } (void)seen_node.insert(node); + TraceGuard guard(std::make_shared(node->debug_info())); AnfNodePtr new_node = Run(func_graph, node); bool change = (new_node != nullptr); if (new_node != nullptr && new_node != node) { diff --git a/mindspore/ccsrc/backend/optimizer/common/pass.h b/mindspore/ccsrc/backend/optimizer/common/pass.h index 8bf4833d63..3fb446784a 100644 --- a/mindspore/ccsrc/backend/optimizer/common/pass.h +++ b/mindspore/ccsrc/backend/optimizer/common/pass.h @@ -20,6 +20,7 @@ #include "ir/anf.h" #include "base/core_ops.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { diff --git a/mindspore/ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc b/mindspore/ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc index 1ebc2f8d67..e08016e9e7 100644 --- a/mindspore/ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/gpu/combine_cast_fusion.cc @@ -99,6 +99,7 @@ bool CastAllFusion::Run(const FuncGraphPtr &graph) { for (size_t idx = 0; idx < cast_list.size(); ++idx) { inputs.push_back(AnfAlgo::GetInputNode(utils::cast(cast_list[idx]), 0)); } + TraceGuard guard(std::make_shared(cast_list[0]->debug_info())); auto cast_all = graph->NewCNode(inputs); auto kernel_info = std::make_shared(); MS_EXCEPTION_IF_NULL(kernel_info); diff --git a/mindspore/ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc b/mindspore/ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc index b7dcc26c97..bd53b96565 100644 --- a/mindspore/ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/gpu/combine_momentum_fusion.cc @@ -104,6 +104,7 @@ bool CombineMomentumFusion::Run(const FuncGraphPtr &graph) { inputs.push_back(AnfAlgo::GetInputNode(utils::cast(mom), i)); } } + TraceGuard guard(std::make_shared(momentums[0]->debug_info())); auto combine_mom = graph->NewCNode(inputs); auto kernel_info = std::make_shared(); MS_EXCEPTION_IF_NULL(kernel_info); diff --git a/mindspore/ccsrc/backend/optimizer/graph_kernel/arithmetic_simplify.cc b/mindspore/ccsrc/backend/optimizer/graph_kernel/arithmetic_simplify.cc index b96f42ff57..d6b16fd363 100644 --- a/mindspore/ccsrc/backend/optimizer/graph_kernel/arithmetic_simplify.cc +++ b/mindspore/ccsrc/backend/optimizer/graph_kernel/arithmetic_simplify.cc @@ -32,7 +32,9 @@ namespace opt { AnfNodePtr NewCNodeWithInfo(const AnfNodePtrList &inputs, const AnfNodePtr &ori_node) { auto func_graph = ori_node->func_graph(); MS_EXCEPTION_IF_NULL(func_graph); + TraceManager::DebugTrace(std::make_shared(ori_node->debug_info())); auto new_cnode = func_graph->NewCNode(inputs); + TraceManager::EndTrace(); new_cnode->set_abstract(ori_node->abstract()); new_cnode->set_kernel_info(std::make_shared()); if (func_graph->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL)) { @@ -687,6 +689,7 @@ void InlineSubgraph(const CNodePtr &kernel_node, const FuncGraphPtr &sub_graph, CNodePtr AddIdentityToEmptyPath(const AnfNodePtr &node, const FuncGraphPtr &sub_graph) { if (node->isa() || node->isa()) { + TraceGuard guard(std::make_shared(node->debug_info())); auto identity_node = sub_graph->NewCNode({NewValueNode(prim::kPrimIdentity), node}); identity_node->set_abstract(node->abstract()); sub_graph->AddNode(identity_node); diff --git a/mindspore/ccsrc/backend/optimizer/graph_kernel/graph_kernel_splitter.cc b/mindspore/ccsrc/backend/optimizer/graph_kernel/graph_kernel_splitter.cc index 2a684a994d..00fbf7ef04 100644 --- a/mindspore/ccsrc/backend/optimizer/graph_kernel/graph_kernel_splitter.cc +++ b/mindspore/ccsrc/backend/optimizer/graph_kernel/graph_kernel_splitter.cc @@ -265,6 +265,7 @@ class AreaGraph { CNodePtr CreateMainCNode(const FuncGraphPtr &main_func_graph, const FuncGraphPtr &sub_func_graph, const std::vector &main_cnodes, const std::unordered_map ¶m_node_map) { + TraceGuard guard(std::make_shared(sub_func_graph->debug_info())); AnfNodePtrList main_cnode_inputs = {NewValueNode(sub_func_graph)}; for (const auto ¶m : sub_func_graph->parameters()) { // assert the param exists. @@ -276,6 +277,7 @@ class AreaGraph { auto idx = NewValueNode(idx_val); idx->set_abstract(std::make_shared(idx_val)); AnfNodePtrList getitem_inputs = {NewValueNode(prim::kPrimTupleGetItem), main_cnodes[input_area], idx}; + TraceGuard g_sub(std::make_shared(main_cnodes[input_area]->debug_info())); auto getitem_node = main_func_graph->NewCNode(getitem_inputs); getitem_node->set_abstract(main_cnodes[input_area]->abstract()); main_cnode_inputs.push_back(getitem_node); diff --git a/mindspore/ccsrc/backend/optimizer/graph_kernel/shape_ops_splitter.cc b/mindspore/ccsrc/backend/optimizer/graph_kernel/shape_ops_splitter.cc index 3efedb82c3..21a31e9719 100644 --- a/mindspore/ccsrc/backend/optimizer/graph_kernel/shape_ops_splitter.cc +++ b/mindspore/ccsrc/backend/optimizer/graph_kernel/shape_ops_splitter.cc @@ -46,6 +46,7 @@ AnfNodePtr CloneCNode(const AnfNodePtr &anf_node) { MS_EXCEPTION_IF_NULL(kernel_graph); auto cnode = anf_node->cast(); MS_EXCEPTION_IF_NULL(cnode); + TraceGuard guard(std::make_shared(cnode->debug_info())); CNodePtr node = kernel_graph->NewCNode(cnode->inputs()); node->set_abstract(cnode->abstract()); node->set_forward(cnode->forward().first, cnode->forward().second); diff --git a/mindspore/ccsrc/backend/optimizer/pass/communication_op_fusion.cc b/mindspore/ccsrc/backend/optimizer/pass/communication_op_fusion.cc index fb2ecf872a..a992217d40 100644 --- a/mindspore/ccsrc/backend/optimizer/pass/communication_op_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/pass/communication_op_fusion.cc @@ -278,6 +278,7 @@ bool CommunicationOpFusion::Run(const FuncGraphPtr &func_graph) { continue; } auto first_node = it.second.communication_op_nodes[0]; + TraceGuard guard(std::make_shared(first_node->debug_info())); if (AnfAlgo::HasNodeAttr(kAttrIndex, first_node) && AnfAlgo::GetNodeAttr(first_node, kAttrIndex) > 0) { std::stable_sort(it.second.communication_op_nodes.begin(), it.second.communication_op_nodes.end(), [](const CNodePtr &a, const CNodePtr &b) { diff --git a/mindspore/ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc b/mindspore/ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc index 53faa131b1..8c6dab79be 100644 --- a/mindspore/ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc +++ b/mindspore/ccsrc/backend/optimizer/pass/replace_node_by_proxy.cc @@ -57,6 +57,7 @@ bool ReplaceNodeByProxy::Run(const FuncGraphPtr &func_graph) { std::vector node_list = TopoSort(func_graph->get_return()); for (auto node : node_list) { if (node != nullptr && node->isa() && AnfAlgo::GetCNodeName(node) == kEmbeddingLookupOpName) { + TraceGuard guard(std::make_shared(node->debug_info())); CNodePtr cnode = node->cast(); auto prim = std::make_shared(kEmbeddingLookupProxyOpName); MS_EXCEPTION_IF_NULL(prim); diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 1bf2f2fffa..fe3e95a13f 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -36,6 +36,7 @@ #include "utils/utils.h" #include "debug/anf_ir_dump.h" #include "mindspore/core/base/base_ref_utils.h" +#include "utils/trace_base.h" #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) #include "ps/worker.h" @@ -992,8 +993,9 @@ std::shared_ptr SessionBasic::ConstructKernelGraph(const FuncGraphP } // Create cnode if (!CreateCNodeOfKernelGraph(node, graph.get())) { - DumpIR("contruct_kernel_graph_fail.ir", func_graph); - MS_LOG_EXCEPTION << "construct func graph " << func_graph->ToString() << "fail!"; + DumpIR("construct_kernel_graph_fail.ir", func_graph); + MS_LOG(EXCEPTION) << "Construct func graph " << func_graph->ToString() << " failed." + << trace::DumpSourceLines(node); } }