!11504 [MS][RDR] add string recorder and record somas infomation

From: @luopengting
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/optimizer/somas/somas.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -49,6 +49,8 @@ class Somas {
   uint8_t *GetNodeOutputPtr(const AnfNodePtr &node, size_t index) const;
   uint8_t *GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const;
 
+  std::string SomasInfo();
+  std::string SomasMemory();
   void DumpSomasInfoIR(const string filename);
   void DumpSomasMemoryIR(const string filename);
 
@@ -121,8 +123,8 @@ class Somas {
 
   bool Assign(const session::KernelGraph *graph);
 
+  std::string Offline();
   void DumpOfflineIR(const string filename);
-  void DumpSomasMemoryPoolInfoIR(const string filename);
   std::string GetSplitName(const string &scope_name) const;
   size_t CalcLowerBound() const;
   void GenGraphStatisticInfo();
@@ -143,9 +145,9 @@ class Somas {
   void UpdateRefOverlapTensorsConflicts();
   void UpdateRefTensorsOffset();
   void UpdateContiguousTensorsOffset(const std::map<size_t, size_t> &contiguous_ref_list_map);
-  void DumpParameters(std::ofstream &ofs) const;
+  void DumpParameters(std::ostringstream &oss) const;
-  void DumpTensors(std::ofstream &ofs) const;
+  void DumpTensors(std::ostringstream &oss) const;
-  void DumpNodes(std::ofstream &ofs) const;
+  void DumpNodes(std::ostringstream &oss) const;
   std::map<size_t, size_t> GetContiguousListContainRefTensor();
   std::map<size_t, size_t> GetRefTensorsInContiguousList();
 };

mindspore/ccsrc/debug/CMakeLists.txt

@@ -8,7 +8,7 @@ set(_DEBUG_SRC_LIST
     "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc"
-    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc"
 )

mindspore/ccsrc/debug/rdr/running_data_recorder.cc

@@ -16,9 +16,9 @@
 #include "debug/rdr/running_data_recorder.h"
 #include <utility>
 #include "debug/rdr/graph_recorder.h"
-#include "debug/rdr/somas_recorder.h"
 #include "debug/rdr/graph_exec_order_recorder.h"
 #include "debug/rdr/recorder_manager.h"
+#include "debug/rdr/string_recorder.h"
 #include "mindspore/core/ir/func_graph.h"
 #include "mindspore/core/ir/anf.h"
 
@@ -75,15 +75,16 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
   return ans;
 }
 
-bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
+bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) {
   std::string submodule_name = std::string(GetSubModuleName(module));
-  SomasRecorderPtr somas_recorder = std::make_shared<SomasRecorder>(submodule_name, tag, somas_ptr, graph_id);
+  StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, tag, data, filename);
-  somas_recorder->GenString();
+  string_recorder->SetFilename(filename);
-  bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder));
+  bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(string_recorder));
   return ans;
 }
 
 void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
+
 #else
 bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
                     const std::string &file_type, int graph_id) {
@@ -107,7 +108,7 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std:
   return false;
 }
 
-bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
+bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) {
   static bool already_printed = false;
   if (already_printed) {
     return false;
@@ -116,6 +117,7 @@ bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const Som
   MS_LOG(WARNING) << "The RDR presently only support linux os.";
   return false;
 }
+
 void TriggerAll() {
   static bool already_printed = false;
   if (already_printed) {

mindspore/ccsrc/debug/rdr/running_data_recorder.h

@@ -13,28 +13,27 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef RUNNING_DATA_RECORDER_H_
+#ifndef MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_
-#define RUNNING_DATA_RECORDER_H_
+#define MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_
 
 #include <vector>
 #include <string>
 #include <memory>
 
-#include "backend/optimizer/somas/somas.h"
 #include "mindspore/core/utils/log_adapter.h"
 namespace mindspore {
 class FuncGraph;
 class CNode;
 using FuncGraphPtr = std::shared_ptr<FuncGraph>;
 using CNodePtr = std::shared_ptr<CNode>;
-using SomasPtr = std::shared_ptr<somas::Somas>;
 namespace RDR {
 bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
                     const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0);
 bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
                           const std::vector<CNodePtr> &&final_exec_order);
-bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id);
+bool RecordString(SubModuleId module, const std::string &tag, const std::string &data,
+                  const std::string &filename = "");
 void TriggerAll();
 }  // namespace RDR
 }  // namespace mindspore
-#endif  // RUNNING_DATA_RECORDER_H_
+#endif  // MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_

mindspore/ccsrc/debug/rdr/somas_recorder.cc

@@ -1,30 +0,0 @@
-/**
- * Copyright 2021 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "debug/rdr/somas_recorder.h"
-#include "backend/optimizer/somas/somas.h"
-namespace mindspore {
-void SomasRecorder::Export() {
-  if (filename_.empty()) {
-    filename_ = directory_ + module_ + "_" + tag_;
-  }
-  std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
-  somas_reuse_util_ptr_->DumpSomasInfoIR(filename);
-  std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
-  somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename);
-}
-
-bool SomasRecorder::GenString() { return true; }
-}  // namespace mindspore

mindspore/ccsrc/debug/rdr/string_recorder.cc

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/string_recorder.h"
+#include <sys/stat.h>
+#include <fstream>
+#include "debug/common.h"
+#include "utils/utils.h"
+#include "mindspore/core/utils/log_adapter.h"
+
+namespace mindspore {
+void StringRecorder::Export() {
+  if (directory_.back() != '/') {
+    directory_ += "/";
+  }
+
+  if (filename_.empty()) {
+    filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt";
+  }
+  std::string file_path = directory_ + filename_;
+
+  auto realpath = Common::GetRealPath(file_path);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path failed. path=" << file_path;
+    return;
+  }
+
+  ChangeFileMode(realpath.value(), S_IRWXU);
+  std::ofstream fout(realpath.value(), std::ofstream::app);
+  if (!fout.is_open()) {
+    MS_LOG(WARNING) << "Open file for saving string failed.";
+    return;
+  }
+  fout << data_;
+  fout.close();
+  // set file mode to read only by user
+  ChangeFileMode(realpath.value(), S_IRUSR);
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/string_recorder.h

@@ -13,34 +13,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef SOMAS_RECORDER_H_
+#ifndef MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_
-#define SOMAS_RECORDER_H_
+#define MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_
 
 #include <string>
-#include <sstream>
 #include <memory>
-#include "debug/rdr/base_recorder.h"
 
+#include "debug/rdr/base_recorder.h"
 namespace mindspore {
-namespace somas {
+class StringRecorder : public BaseRecorder {
-class Somas;
-using SomasPtr = std::shared_ptr<Somas>;
-}  // namespace somas
-class SomasRecorder : public BaseRecorder {
  public:
-  SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {}
+  StringRecorder() : BaseRecorder() {}
-  SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr,
+  StringRecorder(const std::string &module, const std::string &tag, const std::string &data,
-                int graph_id)
+                 const std::string &file_type)
-      : BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {}
+      : BaseRecorder(module, tag), data_(data) {}
+  ~StringRecorder() {}
   void SetModule(const std::string &module) { module_ = module; }
-  void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; }
+  void SetFilename(const std::string &filename) { filename_ = filename; }
-  bool GenString();
   virtual void Export();
 
  private:
-  somas::SomasPtr somas_reuse_util_ptr_;
+  std::string data_;
-  int graph_id_{0};
 };
-using SomasRecorderPtr = std::shared_ptr<SomasRecorder>;
+using StringRecorderPtr = std::shared_ptr<StringRecorder>;
 }  // namespace mindspore
-#endif  // SOMAS_RECORDER_H
+#endif  // MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_

mindspore/ccsrc/runtime/device/memory_manager.cc

@@ -17,6 +17,7 @@
 #include "runtime/device/memory_manager.h"
 #include <string>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "debug/rdr/running_data_recorder.h"
 #include "utils/ms_context.h"
 
 using mindspore::memreuse::BestFitMemReuse;
@@ -69,6 +70,16 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) {
 
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);
+#ifdef ENABLE_DUMP_IR
+  SubModuleId module = SubModuleId::SM_OPTIMIZER;
+  std::string tag = "somas";
+
+  std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir";
+  mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename);
+
+  filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir";
+  mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename);
+#endif
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
   auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH);
   if (save_graphs_path.empty()) {