diff --git a/mindspore/ccsrc/debug/env_config_parser.cc b/mindspore/ccsrc/debug/env_config_parser.cc index 15f3d7c61a..f4c50c6816 100644 --- a/mindspore/ccsrc/debug/env_config_parser.cc +++ b/mindspore/ccsrc/debug/env_config_parser.cc @@ -104,6 +104,8 @@ void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) { return; } + has_rdr_setting_ = true; + auto rdr_enable = CheckJsonKeyExist(*rdr_setting, kRdrSettings, kEnable); if (rdr_enable.has_value()) { ParseRdrEnable(**rdr_enable); diff --git a/mindspore/ccsrc/debug/env_config_parser.h b/mindspore/ccsrc/debug/env_config_parser.h index a3744a091e..f835223b5e 100644 --- a/mindspore/ccsrc/debug/env_config_parser.h +++ b/mindspore/ccsrc/debug/env_config_parser.h @@ -27,11 +27,14 @@ class EnvConfigParser { public: static EnvConfigParser &GetInstance() { static EnvConfigParser instance; + instance.Parse(); return instance; } void Parse(); + std::string config_path() const { return config_file_; } + bool has_rdr_setting() const { return has_rdr_setting_; } bool rdr_enabled() const { return rdr_enabled_; } std::string rdr_path() const { return rdr_path_; } @@ -42,7 +45,9 @@ class EnvConfigParser { std::mutex lock_; std::string config_file_{""}; bool already_parsed_{false}; + bool rdr_enabled_{false}; + bool has_rdr_setting_{false}; std::string rdr_path_{"./rdr/"}; std::string GetIfstreamString(const std::ifstream &ifstream); diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.cc b/mindspore/ccsrc/debug/rdr/base_recorder.cc index 52e97a11d0..85989ea1af 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/base_recorder.cc @@ -39,13 +39,13 @@ void BaseRecorder::SetFilename(const std::string &filename) { std::optional BaseRecorder::GetFileRealPath(const std::string &suffix) { if (filename_.empty()) { - filename_ = module_ + "_" + tag_; + filename_ = module_ + delimiter_ + tag_; if (!suffix.empty()) { - filename_ += "_" + suffix; + filename_ += delimiter_ + suffix; } - filename_ += "_" + timestamp_; + filename_ += delimiter_ + timestamp_; } else if (!suffix.empty()) { - filename_ += "_" + suffix; + filename_ += delimiter_ + suffix; } std::string file_path = directory_ + filename_; auto realpath = Common::GetRealPath(file_path); diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.h b/mindspore/ccsrc/debug/rdr/base_recorder.h index 300b116e8a..12f019a879 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.h +++ b/mindspore/ccsrc/debug/rdr/base_recorder.h @@ -65,6 +65,7 @@ class BaseRecorder { void SetDirectory(const std::string &directory); void SetFilename(const std::string &filename); + void SetModule(const std::string &module) { module_ = module; } virtual void Export() {} protected: @@ -73,6 +74,7 @@ class BaseRecorder { std::string directory_; std::string filename_; std::string timestamp_; // year,month,day,hour,minute,second + std::string delimiter_{"."}; }; using BaseRecorderPtr = std::shared_ptr; } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc index d3b66e320b..d9d2c9afd7 100644 --- a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc @@ -46,11 +46,11 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector } // namespace void GraphExecOrderRecorder::Export() { - auto realpath = GetFileRealPath(); + auto realpath = GetFileRealPath(std::to_string(graph_id_)); if (!realpath.has_value()) { return; } - std::string real_file_path = realpath.value() + std::to_string(graph_id_); + std::string real_file_path = realpath.value() + ".txt"; DumpGraphExeOrder(real_file_path, exec_order_); } } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h index ce707935e1..1b5b44f8ad 100644 --- a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h @@ -30,7 +30,6 @@ class GraphExecOrderRecorder : public BaseRecorder { GraphExecOrderRecorder(const std::string &module, const std::string &tag, const std::vector &final_exec_order, int graph_id) : BaseRecorder(module, tag), exec_order_(final_exec_order), graph_id_(graph_id) {} - void SetModule(const std::string &module) { module_ = module; } void SetExecOrder(const std::vector &final_exec_order) { exec_order_ = final_exec_order; } virtual void Export(); diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc index 1e81834901..a66ac61d7b 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -65,6 +65,7 @@ void GraphRecorder::Export() { } std::string suffix = graph_id >= 0 ? std::to_string(graph_id) : ""; auto tmp_realpath = GetFileRealPath(suffix); + if (!tmp_realpath.has_value()) { return; } diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.h b/mindspore/ccsrc/debug/rdr/graph_recorder.h index cda58039b0..bf48157943 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.h @@ -31,7 +31,6 @@ class GraphRecorder : public BaseRecorder { const std::string &file_type) : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type) {} ~GraphRecorder() {} - void SetModule(const std::string &module) { module_ = module; } void SetGraphType(const std::string &file_type) { graph_type_ = file_type; } void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; } void SetDumpFlag(bool full_name) { full_name_ = full_name; } diff --git a/mindspore/ccsrc/debug/rdr/recorder_manager.cc b/mindspore/ccsrc/debug/rdr/recorder_manager.cc index 19f087c424..ab81f39d7b 100644 --- a/mindspore/ccsrc/debug/rdr/recorder_manager.cc +++ b/mindspore/ccsrc/debug/rdr/recorder_manager.cc @@ -21,7 +21,35 @@ #include "mindspore/core/ir/func_graph.h" namespace mindspore { + +void RecorderManager::UpdateRdrEnable() { + static bool updated = false; + if (updated) { + return; + } + auto &config_parser = mindspore::EnvConfigParser::GetInstance(); + rdr_enable_ = config_parser.rdr_enabled(); + if (config_parser.has_rdr_setting()) { +#ifdef __linux__ + if (!rdr_enable_) { + MS_LOG(WARNING) << "Please set the 'enable' as true using 'rdr' setting in file '" << config_parser.config_path() + << "' if you want to use RDR."; + } +#else + if (rdr_enable_) { + MS_LOG(WARNING) << "The RDR only supports linux os currently."; + } + rdr_enable_ = false; +#endif + } + updated = true; +} + bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) { + if (!rdr_enable_) { + return false; + } + if (recorder == nullptr) { MS_LOG(ERROR) << "register recorder module with nullptr."; return false; @@ -33,10 +61,7 @@ bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) { } void RecorderManager::TriggerAll() { - auto &config_parser_ptr = mindspore::EnvConfigParser::GetInstance(); - config_parser_ptr.Parse(); - if (!config_parser_ptr.rdr_enabled()) { - MS_LOG(INFO) << "RDR is not enable."; + if (!rdr_enable_) { return; } diff --git a/mindspore/ccsrc/debug/rdr/recorder_manager.h b/mindspore/ccsrc/debug/rdr/recorder_manager.h index a78c21da32..760f2e83a4 100644 --- a/mindspore/ccsrc/debug/rdr/recorder_manager.h +++ b/mindspore/ccsrc/debug/rdr/recorder_manager.h @@ -31,9 +31,12 @@ class RecorderManager { public: static RecorderManager &Instance() { static RecorderManager manager; + manager.UpdateRdrEnable(); return manager; } + void UpdateRdrEnable(); + bool RdrEnable() const { return rdr_enable_; } bool RecordObject(const BaseRecorderPtr &recorder); void TriggerAll(); void ClearAll(); @@ -42,6 +45,8 @@ class RecorderManager { RecorderManager() {} ~RecorderManager() {} + bool rdr_enable_{false}; + mutable std::mutex mtx_; // module, BaserRecorderPtrList std::unordered_map recorder_container_; diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc index 8b3e0cea98..2e8ae1e087 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -65,6 +65,9 @@ namespace RDR { #ifdef ENABLE_D bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, const std::vector &task_debug_info_list, int graph_id) { + if (!mindspore::RecorderManager::Instance().RdrEnable()) { + return false; + } std::string submodule_name = std::string(GetSubModuleName(module)); TaskDebugInfoRecorderPtr task_debug_info_recorder = std::make_shared(submodule_name, tag, task_debug_info_list, graph_id); @@ -73,9 +76,11 @@ bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, } #endif // ENABLE_D -#ifdef __linux__ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type) { + if (!mindspore::RecorderManager::Instance().RdrEnable()) { + return false; + } std::string submodule_name = std::string(GetSubModuleName(module)); GraphRecorderPtr graph_recorder = std::make_shared(submodule_name, tag, graph, file_type); graph_recorder->SetDumpFlag(full_name); @@ -85,6 +90,9 @@ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const Func bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, const std::vector &final_exec_order, int graph_id) { + if (!mindspore::RecorderManager::Instance().RdrEnable()) { + return false; + } std::string submodule_name = std::string(GetSubModuleName(module)); GraphExecOrderRecorderPtr graph_exec_order_recorder = std::make_shared(submodule_name, tag, final_exec_order, graph_id); @@ -93,6 +101,9 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, } bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { + if (!mindspore::RecorderManager::Instance().RdrEnable()) { + return false; + } std::string submodule_name = std::string(GetSubModuleName(module)); StringRecorderPtr string_recorder = std::make_shared(submodule_name, tag, data, filename); string_recorder->SetFilename(filename); @@ -102,6 +113,9 @@ bool RecordString(SubModuleId module, const std::string &tag, const std::string bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id, const std::vector &exec_order) { + if (!mindspore::RecorderManager::Instance().RdrEnable()) { + return false; + } std::string submodule_name = std::string(GetSubModuleName(module)); StreamExecOrderRecorderPtr stream_exec_order_recorder = std::make_shared(submodule_name, tag, graph_id, exec_order); @@ -113,68 +127,5 @@ void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } void ClearAll() { mindspore::RecorderManager::Instance().ClearAll(); } -#else -bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, - const std::string &file_type) { - static bool already_printed = false; - std::string submodule_name = std::string(GetSubModuleName(module)); - if (already_printed) { - return false; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os " << submodule_name; - return false; -} - -bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, - const std::vector &final_exec_order, int graph_id) { - static bool already_printed = false; - if (already_printed) { - return false; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os."; - return false; -} - -bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { - static bool already_printed = false; - if (already_printed) { - return false; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os."; - return false; -} - -bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id, - const std::vector &exec_order) { - static bool already_printed = false; - if (already_printed) { - return false; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os."; - return false; -} - -void TriggerAll() { - static bool already_printed = false; - if (already_printed) { - return; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os."; -} - -void ClearAll() { - static bool already_printed = false; - if (already_printed) { - return; - } - already_printed = true; - MS_LOG(WARNING) << "The RDR presently only support linux os."; -} -#endif // __linux__ } // namespace RDR } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h b/mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h index 8cfe0eb6e2..e16014c35f 100644 --- a/mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h +++ b/mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h @@ -98,7 +98,6 @@ class StreamExecOrderRecorder : public BaseRecorder { exec_order_.push_back(std::move(exec_node_ptr)); } } - void SetModule(const std::string &module) { module_ = module; } virtual void Export(); private: diff --git a/mindspore/ccsrc/debug/rdr/string_recorder.h b/mindspore/ccsrc/debug/rdr/string_recorder.h index a6ba4b0e1f..ba4468a1db 100644 --- a/mindspore/ccsrc/debug/rdr/string_recorder.h +++ b/mindspore/ccsrc/debug/rdr/string_recorder.h @@ -26,14 +26,14 @@ class StringRecorder : public BaseRecorder { StringRecorder() : BaseRecorder() {} StringRecorder(const std::string &module, const std::string &tag, const std::string &data, const std::string &filename) - : BaseRecorder(module, tag), data_(data), filename_(filename) {} + : BaseRecorder(module, tag), data_(data) { + SetFilename(filename); + } ~StringRecorder() {} - void SetModule(const std::string &module) { module_ = module; } virtual void Export(); private: std::string data_; - std::string filename_; }; using StringRecorderPtr = std::shared_ptr; } // namespace mindspore diff --git a/mindspore/context.py b/mindspore/context.py index e15a8f4cfc..78c7d19a62 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -258,8 +258,8 @@ class _Context: def set_env_config_path(self, env_config_path): """Check and set env_config_path.""" if not self._context_handle.enable_dump_ir(): - raise ValueError("The 'env_config_path' is not supported, please turn on ENABLE_DUMP_IR " - "and recompile source to enable it.") + raise ValueError("The 'env_config_path' is not supported, please enable ENABLE_DUMP_IR " + "with '-D on' and recompile source.") env_config_path = os.path.realpath(env_config_path) if not os.path.isfile(env_config_path): raise ValueError("The %r set by 'env_config_path' should be an existing json file." % env_config_path)