From f00032145fd18ce688470768a51c2cff499e3ff0 Mon Sep 17 00:00:00 2001 From: louei5 Date: Thu, 28 Jan 2021 19:50:42 +0800 Subject: [PATCH] Add timestamp into filename of saving recorder file and change DumpIR method --- .../ascend/ascend_backend_optimization.cc | 7 +- mindspore/ccsrc/backend/session/executor.cc | 2 +- .../ccsrc/backend/session/session_basic.cc | 2 +- mindspore/ccsrc/debug/anf_ir_dump.cc | 54 ++++++++++++++- mindspore/ccsrc/debug/anf_ir_dump.h | 5 +- mindspore/ccsrc/debug/rdr/base_recorder.h | 23 +++++-- .../debug/rdr/graph_exec_order_recorder.h | 6 +- mindspore/ccsrc/debug/rdr/graph_recorder.cc | 67 ++++++++++++++++++- mindspore/ccsrc/debug/rdr/graph_recorder.h | 2 + mindspore/ccsrc/debug/rdr/recorder_manager.h | 6 +- .../ccsrc/debug/rdr/running_data_recorder.cc | 5 +- .../ccsrc/debug/rdr/running_data_recorder.h | 8 +-- 12 files changed, 160 insertions(+), 27 deletions(-) diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc b/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc index 4f4829fed9..3654459d85 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc @@ -266,7 +266,7 @@ void AscendBackendIRFusionOptimization(const std::shared_ptrget_param(MS_CTX_SAVE_GRAPHS_FLAG); #ifdef ENABLE_DUMP_IR std::string tag = "before_hwopt"; - mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph); + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, false, ".ir;.pb"); #endif if (save_graphs) { std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir"; @@ -388,10 +388,11 @@ void AscendBackendOptimization(const std::shared_ptr &kern kernel_graph->SetExecOrderByDefault(); #ifdef ENABLE_DUMP_IR std::string tag = "hwopt_d_end"; - mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph); + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, true, ".ir;.pb"); const std::vector &exec_order = kernel_graph->execution_order(); + std::vector graph_exec_order(exec_order); tag = "graph_exec_order"; - mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(exec_order)); + mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(graph_exec_order)); #endif if (save_graphs) { std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir"; diff --git a/mindspore/ccsrc/backend/session/executor.cc b/mindspore/ccsrc/backend/session/executor.cc index e4f53d4705..8fc9eeb4a9 100644 --- a/mindspore/ccsrc/backend/session/executor.cc +++ b/mindspore/ccsrc/backend/session/executor.cc @@ -120,7 +120,7 @@ void RunGraphTask::Run() { #ifdef ENABLE_DUMP_IR std::string tag = "run_graph"; std::string file_type = ".ir;.pb"; - mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type); + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, file_type); #endif graph->ResetGraphRunningStatus(); try { diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 2974b93ca5..1955acf15a 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -1094,7 +1094,7 @@ std::shared_ptr SessionBasic::ConstructKernelGraph(const FuncGraphP #ifdef ENABLE_DUMP_IR std::string tag = "constructed_kernel_graph"; std::string file_type = ".ir;.pb"; - mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type); + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, file_type); #endif front_backend_graph_map_[func_graph] = graph; MS_LOG(INFO) << "Create graph: " << graph->graph_id(); diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc index 7c593ae9aa..a8a09931f8 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.cc +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -596,6 +596,49 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu // set file mode to read only by user ChangeFileMode(realpath.value(), S_IRUSR); } + +void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool dump_full_name, + LocDumpMode dump_location) { + GetEnvDumpIrLineLevel(&dump_location); + if (graph == nullptr) { + return; + } + auto path = AddGlobalId(filename); + auto realpath = Common::GetRealPath(path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed. path=" << path; + return; + } + + ChangeFileMode(realpath.value(), S_IRWXU); + std::ofstream fout(realpath.value()); + std::ostringstream buffer; + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open dump file '" << realpath.value() << "' failed!"; + return; + } + + auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude); + OrderedMap para_map; + // dump global info + DumpGlobalInfoEntry(graph, buffer); + int32_t total_para = DumpParams(graph, buffer, ¶_map); + + OrderedMap> sub_graphs; + // dump ir in each sub graph + DumpIRInSubgraph(nodes, ¶_map, &sub_graphs, total_para, dump_full_name, dump_location); + + // output global info + fout << buffer.str() << std::endl; + + // output each sub graph + DumpSubgraph(&sub_graphs, graph, ¶_map, fout); + + fout.close(); + // set file mode to read only by user + ChangeFileMode(realpath.value(), S_IRUSR); +} + #else void DumpIR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) { static bool already_printed = false; @@ -606,5 +649,14 @@ void DumpIR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) { MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " << "please recompile source to enable it. See help of building script."; } +void DumpIRForRDR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " + << "please recompile source to enable it. See help of building script."; +} #endif } // namespace mindspore diff --git a/mindspore/ccsrc/debug/anf_ir_dump.h b/mindspore/ccsrc/debug/anf_ir_dump.h index df6eb37723..90de5fa22f 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.h +++ b/mindspore/ccsrc/debug/anf_ir_dump.h @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,9 @@ constexpr char PARALLEL_STRATEGY[] = "strategy"; void DumpIR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false, LocDumpMode dump_location = kOff); void PrintInputAndOutputInferType(std::ostringstream &buffer, const AnfNodePtr &nd); + +void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false, + LocDumpMode dump_location = kOff); const std::string ToShortString(const TypeId &typeId); } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.h b/mindspore/ccsrc/debug/rdr/base_recorder.h index d110e889b4..5d3da0348d 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.h +++ b/mindspore/ccsrc/debug/rdr/base_recorder.h @@ -18,22 +18,35 @@ #include #include +#include +#include +#include #include "debug/env_config_parser.h" - namespace mindspore { class BaseRecorder { public: BaseRecorder() : module_(""), tag_(""), directory_(""), filename_(""), timestamp_("") {} - BaseRecorder(const std::string &module, const std::string &tag) - : module_(module), tag_(tag), directory_(""), filename_(""), timestamp_("") { + BaseRecorder(const std::string &module, const std::string &tag) : module_(module), tag_(tag), filename_("") { auto &config_parser_ptr = mindspore::EnvConfigParser::GetInstance(); config_parser_ptr.Parse(); directory_ = config_parser_ptr.rdr_path(); + + auto sys_time = std::chrono::system_clock::now(); + auto t_time = std::chrono::system_clock::to_time_t(sys_time); + std::tm *l_time = std::localtime(&t_time); + if (l_time == nullptr) { + timestamp_ = ""; + } else { + std::stringstream ss; + ss << std::put_time(l_time, "%Y%m%d%H%M%S"); + timestamp_ = ss.str(); + } } ~BaseRecorder() {} - std::string GetModule() { return module_; } - std::string GetTag() { return tag_; } + std::string GetModule() const { return module_; } + std::string GetTag() const { return tag_; } + std::string GetTimeStamp() const { return timestamp_; } void SetDirectory(const std::string &directory) { directory_ = directory; } diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h index 652833a2b4..650d94ed50 100644 --- a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef GRAPH_EXEC_ORDER_RENDER_H_ -#define GRAPH_EXEC_ORDER_RENDER_H_ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_ #include #include #include @@ -39,4 +39,4 @@ class GraphExecOrderRecorder : public BaseRecorder { }; using GraphExecOrderRecorderPtr = std::shared_ptr; } // namespace mindspore -#endif // GRAPH_EXEC_ORDER_RENDER_H +#endif // MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc index b61b93a3ab..874e51fe72 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -22,6 +22,59 @@ #include "debug/dump_proto.h" namespace mindspore { +namespace dumper { +#ifdef ENABLE_DUMP_IR +void DumpIRProto(const std::string &filename, const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + MS_LOG(ERROR) << "Func graph is nullptr"; + return; + } + + if (filename.size() > PATH_MAX) { + MS_LOG(ERROR) << "File path " << filename << " is too long."; + return; + } + char real_path[PATH_MAX] = {0}; + char *real_path_ret = nullptr; +#if defined(_WIN32) || defined(_WIN64) + real_path_ret = _fullpath(real_path, filename.c_str(), PATH_MAX); +#else + real_path_ret = realpath(filename.c_str(), real_path); +#endif + if (nullptr == real_path_ret) { + MS_LOG(DEBUG) << "dir " << filename << " does not exit."; + } else { + std::string path_string = real_path; + if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { + MS_LOG(ERROR) << "Modify file:" << real_path << " to rw fail."; + return; + } + } + + // write to pb file + std::ofstream ofs(real_path); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file '" << real_path << "' failed!"; + return; + } + ofs << GetFuncGraphProtoString(func_graph); + ofs.close(); + // set file mode to read only by user + ChangeFileMode(real_path, S_IRUSR); +} +#else +void DumpIRProto(const FuncGraphPtr &, const std::string &) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR in protobuf format is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif +} // namespace dumper + void GraphRecorder::Export() { bool save_flag = false; if (filename_.empty()) { @@ -29,15 +82,23 @@ void GraphRecorder::Export() { } if (graph_type_.find(".dat") != std::string::npos) { save_flag = true; - ExportIR(filename_ + ".dat", std::to_string(id_), func_graph_); // saving *.dat file + AnfExporter exporter(std::to_string(id_)); + std::string filename = filename_ + ".dat"; + ChangeFileMode(filename, S_IRWXU); + exporter.ExportFuncGraph(filename, func_graph_); + ChangeFileMode(filename, S_IRUSR); } if (graph_type_.find(".ir") != std::string::npos) { save_flag = true; - DumpIR(filename_ + ".ir", func_graph_); // saving *.ir file + if (full_name_) { + DumpIRForRDR(filename_ + ".ir", func_graph_, true, kTopStack); + } else { + DumpIRForRDR(filename_ + ".ir", func_graph_, false, kOff); + } } if (graph_type_.find(".pb") != std::string::npos) { save_flag = true; - DumpIRProto(func_graph_, filename_); // save *.pb file + dumper::DumpIRProto(filename_ + ".pb", func_graph_); // save *.pb file } if (!save_flag) { MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_; diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.h b/mindspore/ccsrc/debug/rdr/graph_recorder.h index c14d49d82d..1d70009744 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.h @@ -35,12 +35,14 @@ class GraphRecorder : public BaseRecorder { void SetModule(const std::string &module) { module_ = module; } void SetGraphType(const std::string &file_type) { graph_type_ = file_type; } void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; } + void SetDumpFlag(bool full_name) { full_name_ = full_name; } void SetNodeId(int id) { id_ = id; } virtual void Export(); private: FuncGraphPtr func_graph_; std::string graph_type_; + bool full_name_{false}; int id_{0}; }; using GraphRecorderPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/debug/rdr/recorder_manager.h b/mindspore/ccsrc/debug/rdr/recorder_manager.h index 632bcc521e..4f4b9c5d38 100644 --- a/mindspore/ccsrc/debug/rdr/recorder_manager.h +++ b/mindspore/ccsrc/debug/rdr/recorder_manager.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MINDSPORE_RECORDER_MANAGER_H_ -#define MINDSPORE_RECORDER_MANAGER_H_ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_ #include #include #include @@ -46,4 +46,4 @@ class RecorderManager { std::unordered_map recorder_container_; }; } // namespace mindspore -#endif // MINDSPORE_RECORDER_MANAGER_H +#endif // MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_ diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc index b9c1a406f3..4e97e04f51 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -58,10 +58,11 @@ static const char *GetSubModuleName(SubModuleId module_id) { } // namespace namespace RDR { #ifdef __linux__ -bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type, int graph_id) { std::string submodule_name = std::string(GetSubModuleName(module)); GraphRecorderPtr graph_recorder = std::make_shared(submodule_name, tag, graph, file_type, graph_id); + graph_recorder->SetDumpFlag(full_name); bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder)); return ans; } @@ -85,7 +86,7 @@ bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const Som void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } #else -bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type, int graph_id) { static bool already_printed = false; std::string submodule_name = std::string(GetSubModuleName(module)); diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.h b/mindspore/ccsrc/debug/rdr/running_data_recorder.h index 36241b99fa..5f4061a328 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.h +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef RUNNING_DATA_RECORDER_H_ -#define RUNNING_DATA_RECORDER_H_ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ #include #include @@ -29,7 +29,7 @@ using FuncGraphPtr = std::shared_ptr; using CNodePtr = std::shared_ptr; using SomasPtr = std::shared_ptr; namespace RDR { -bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0); bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, const std::vector &&final_exec_order); @@ -37,4 +37,4 @@ bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const Som void TriggerAll(); } // namespace RDR } // namespace mindspore -#endif // RUNNING_DATA_RECORDER_H_ +#endif // MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_