!11698 Add timestamp for RDR module

From: @louie5
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc

@@ -266,7 +266,7 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr<session::KernelGrap
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
 #ifdef ENABLE_DUMP_IR
   std::string tag = "before_hwopt";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, false, ".ir;.pb");
 #endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
@@ -388,10 +388,11 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern
   kernel_graph->SetExecOrderByDefault();
 #ifdef ENABLE_DUMP_IR
   std::string tag = "hwopt_d_end";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, true, ".ir;.pb");
   const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order();
+  std::vector<CNodePtr> graph_exec_order(exec_order);
   tag = "graph_exec_order";
-  mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(exec_order));
+  mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(graph_exec_order));
 #endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";

mindspore/ccsrc/backend/session/executor.cc

@@ -120,7 +120,7 @@ void RunGraphTask::Run() {
 #ifdef ENABLE_DUMP_IR
   std::string tag = "run_graph";
   std::string file_type = ".ir;.pb";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, file_type);
 #endif
   graph->ResetGraphRunningStatus();
   try {

mindspore/ccsrc/backend/session/session_basic.cc

@@ -1094,7 +1094,7 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
 #ifdef ENABLE_DUMP_IR
   std::string tag = "constructed_kernel_graph";
   std::string file_type = ".ir;.pb";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, file_type);
 #endif
   front_backend_graph_map_[func_graph] = graph;
   MS_LOG(INFO) << "Create graph: " << graph->graph_id();

mindspore/ccsrc/debug/anf_ir_dump.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -596,6 +596,49 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
   // set file mode to read only by user
   ChangeFileMode(realpath.value(), S_IRUSR);
 }
+
+void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool dump_full_name,
+                  LocDumpMode dump_location) {
+  GetEnvDumpIrLineLevel(&dump_location);
+  if (graph == nullptr) {
+    return;
+  }
+  auto path = AddGlobalId(filename);
+  auto realpath = Common::GetRealPath(path);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path failed. path=" << path;
+    return;
+  }
+
+  ChangeFileMode(realpath.value(), S_IRWXU);
+  std::ofstream fout(realpath.value());
+  std::ostringstream buffer;
+  if (!fout.is_open()) {
+    MS_LOG(ERROR) << "Open dump file '" << realpath.value() << "' failed!";
+    return;
+  }
+
+  auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude);
+  OrderedMap<AnfNodePtr, int32_t> para_map;
+  // dump global info
+  DumpGlobalInfoEntry(graph, buffer);
+  int32_t total_para = DumpParams(graph, buffer, &para_map);
+
+  OrderedMap<FuncGraphPtr, std::shared_ptr<SubGraphIRInfo>> sub_graphs;
+  // dump ir in each sub graph
+  DumpIRInSubgraph(nodes, &para_map, &sub_graphs, total_para, dump_full_name, dump_location);
+
+  // output global info
+  fout << buffer.str() << std::endl;
+
+  // output each sub graph
+  DumpSubgraph(&sub_graphs, graph, &para_map, fout);
+
+  fout.close();
+  // set file mode to read only by user
+  ChangeFileMode(realpath.value(), S_IRUSR);
+}
+
 #else
 void DumpIR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) {
   static bool already_printed = false;
@@ -606,5 +649,14 @@ void DumpIR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) {
   MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, "
                   << "please recompile source to enable it. See help of building script.";
 }
+void DumpIRForRDR(const std::string &, const FuncGraphPtr &, bool, LocDumpMode) {
+  static bool already_printed = false;
+  if (already_printed) {
+    return;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, "
+                  << "please recompile source to enable it. See help of building script.";
+}
 #endif
 }  // namespace mindspore

mindspore/ccsrc/debug/anf_ir_dump.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,9 @@ constexpr char PARALLEL_STRATEGY[] = "strategy";
 void DumpIR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
             LocDumpMode dump_location = kOff);
 void PrintInputAndOutputInferType(std::ostringstream &buffer, const AnfNodePtr &nd);
+
+void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
+                  LocDumpMode dump_location = kOff);
 const std::string ToShortString(const TypeId &typeId);
 }  // namespace mindspore
 

mindspore/ccsrc/debug/rdr/base_recorder.h

@@ -18,22 +18,35 @@
 
 #include <memory>
 #include <string>
+#include <sstream>
+#include <chrono>
+#include <iomanip>
 #include "debug/env_config_parser.h"
-
 namespace mindspore {
 class BaseRecorder {
  public:
   BaseRecorder() : module_(""), tag_(""), directory_(""), filename_(""), timestamp_("") {}
-  BaseRecorder(const std::string &module, const std::string &tag)
+  BaseRecorder(const std::string &module, const std::string &tag) : module_(module), tag_(tag), filename_("") {
-      : module_(module), tag_(tag), directory_(""), filename_(""), timestamp_("") {
     auto &config_parser_ptr = mindspore::EnvConfigParser::GetInstance();
     config_parser_ptr.Parse();
     directory_ = config_parser_ptr.rdr_path();
+
+    auto sys_time = std::chrono::system_clock::now();
+    auto t_time = std::chrono::system_clock::to_time_t(sys_time);
+    std::tm *l_time = std::localtime(&t_time);
+    if (l_time == nullptr) {
+      timestamp_ = "";
+    } else {
+      std::stringstream ss;
+      ss << std::put_time(l_time, "%Y%m%d%H%M%S");
+      timestamp_ = ss.str();
+    }
   }
   ~BaseRecorder() {}
 
-  std::string GetModule() { return module_; }
+  std::string GetModule() const { return module_; }
-  std::string GetTag() { return tag_; }
+  std::string GetTag() const { return tag_; }
+  std::string GetTimeStamp() const { return timestamp_; }
 
   void SetDirectory(const std::string &directory) { directory_ = directory; }
 

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef GRAPH_EXEC_ORDER_RENDER_H_
+#ifndef MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_
-#define GRAPH_EXEC_ORDER_RENDER_H_
+#define MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_
 #include <vector>
 #include <string>
 #include <memory>
@@ -39,4 +39,4 @@ class GraphExecOrderRecorder : public BaseRecorder {
 };
 using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>;
 }  // namespace mindspore
-#endif  // GRAPH_EXEC_ORDER_RENDER_H
+#endif  // MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_EXEC_ORDER_RECORDER_H_

mindspore/ccsrc/debug/rdr/graph_recorder.cc

@@ -22,6 +22,59 @@
 #include "debug/dump_proto.h"
 
 namespace mindspore {
+namespace dumper {
+#ifdef ENABLE_DUMP_IR
+void DumpIRProto(const std::string &filename, const FuncGraphPtr &func_graph) {
+  if (func_graph == nullptr) {
+    MS_LOG(ERROR) << "Func graph is nullptr";
+    return;
+  }
+
+  if (filename.size() > PATH_MAX) {
+    MS_LOG(ERROR) << "File path " << filename << " is too long.";
+    return;
+  }
+  char real_path[PATH_MAX] = {0};
+  char *real_path_ret = nullptr;
+#if defined(_WIN32) || defined(_WIN64)
+  real_path_ret = _fullpath(real_path, filename.c_str(), PATH_MAX);
+#else
+  real_path_ret = realpath(filename.c_str(), real_path);
+#endif
+  if (nullptr == real_path_ret) {
+    MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
+  } else {
+    std::string path_string = real_path;
+    if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {
+      MS_LOG(ERROR) << "Modify file:" << real_path << " to rw fail.";
+      return;
+    }
+  }
+
+  // write to pb file
+  std::ofstream ofs(real_path);
+  if (!ofs.is_open()) {
+    MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
+    return;
+  }
+  ofs << GetFuncGraphProtoString(func_graph);
+  ofs.close();
+  // set file mode to read only by user
+  ChangeFileMode(real_path, S_IRUSR);
+}
+#else
+void DumpIRProto(const FuncGraphPtr &, const std::string &) {
+  static bool already_printed = false;
+  if (already_printed) {
+    return;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The functionality of dumping function graph IR in protobuf format is disabled, "
+                  << "please recompile source to enable it. See help of building script.";
+}
+#endif
+}  // namespace dumper
+
 void GraphRecorder::Export() {
   bool save_flag = false;
   if (filename_.empty()) {
@@ -29,15 +82,23 @@ void GraphRecorder::Export() {
   }
   if (graph_type_.find(".dat") != std::string::npos) {
     save_flag = true;
-    ExportIR(filename_ + ".dat", std::to_string(id_), func_graph_);  // saving  *.dat file
+    AnfExporter exporter(std::to_string(id_));
+    std::string filename = filename_ + ".dat";
+    ChangeFileMode(filename, S_IRWXU);
+    exporter.ExportFuncGraph(filename, func_graph_);
+    ChangeFileMode(filename, S_IRUSR);
   }
   if (graph_type_.find(".ir") != std::string::npos) {
     save_flag = true;
-    DumpIR(filename_ + ".ir", func_graph_);  // saving *.ir file
+    if (full_name_) {
+      DumpIRForRDR(filename_ + ".ir", func_graph_, true, kTopStack);
+    } else {
+      DumpIRForRDR(filename_ + ".ir", func_graph_, false, kOff);
+    }
   }
   if (graph_type_.find(".pb") != std::string::npos) {
     save_flag = true;
-    DumpIRProto(func_graph_, filename_);  // save *.pb file
+    dumper::DumpIRProto(filename_ + ".pb", func_graph_);  // save *.pb file
   }
   if (!save_flag) {
     MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_;

mindspore/ccsrc/debug/rdr/graph_recorder.h

@@ -35,12 +35,14 @@ class GraphRecorder : public BaseRecorder {
   void SetModule(const std::string &module) { module_ = module; }
   void SetGraphType(const std::string &file_type) { graph_type_ = file_type; }
   void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; }
+  void SetDumpFlag(bool full_name) { full_name_ = full_name; }
   void SetNodeId(int id) { id_ = id; }
   virtual void Export();
 
  private:
   FuncGraphPtr func_graph_;
   std::string graph_type_;
+  bool full_name_{false};
   int id_{0};
 };
 using GraphRecorderPtr = std::shared_ptr<GraphRecorder>;

mindspore/ccsrc/debug/rdr/recorder_manager.h

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef MINDSPORE_RECORDER_MANAGER_H_
+#ifndef MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_
-#define MINDSPORE_RECORDER_MANAGER_H_
+#define MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_
 #include <vector>
 #include <string>
 #include <sstream>
@@ -46,4 +46,4 @@ class RecorderManager {
   std::unordered_map<std::string, BaseRecorderPtrList> recorder_container_;
 };
 }  // namespace mindspore
-#endif  // MINDSPORE_RECORDER_MANAGER_H
+#endif  // MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_

mindspore/ccsrc/debug/rdr/running_data_recorder.cc

@@ -58,10 +58,11 @@ static const char *GetSubModuleName(SubModuleId module_id) {
 }  // namespace
 namespace RDR {
 #ifdef __linux__
-bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
                     const std::string &file_type, int graph_id) {
   std::string submodule_name = std::string(GetSubModuleName(module));
   GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, tag, graph, file_type, graph_id);
+  graph_recorder->SetDumpFlag(full_name);
   bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder));
   return ans;
 }
@@ -86,7 +87,7 @@ bool RecordString(SubModuleId module, const std::string &tag, const std::string
 void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
 
 #else
-bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
                     const std::string &file_type, int graph_id) {
   static bool already_printed = false;
   std::string submodule_name = std::string(GetSubModuleName(module));

mindspore/ccsrc/debug/rdr/running_data_recorder.h

@@ -27,7 +27,7 @@ class CNode;
 using FuncGraphPtr = std::shared_ptr<FuncGraph>;
 using CNodePtr = std::shared_ptr<CNode>;
 namespace RDR {
-bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
                     const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0);
 bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
                           const std::vector<CNodePtr> &&final_exec_order);