Adding RDR tool for locating bugs, e.g. run task error.

pull/11461/head
louei5 4 years ago
parent 54b8d53780
commit ac02213521

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -122,7 +122,9 @@
#include "utils/config_manager.h"
#include "debug/anf_ir_dump.h"
#include "debug/dump_proto.h"
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/running_data_recorder.h"
#endif
namespace mindspore {
namespace opt {
namespace {
@ -262,6 +264,10 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr<session::KernelGrap
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
#ifdef ENABLE_DUMP_IR
std::string tag = "before_hwopt";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
#endif
if (save_graphs) {
std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
DumpIR(file_name, kernel_graph);
@ -380,7 +386,13 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern
optimizer2->AddPassManager(other2_pm);
(void)optimizer2->Optimize(kernel_graph);
kernel_graph->SetExecOrderByDefault();
#ifdef ENABLE_DUMP_IR
std::string tag = "hwopt_d_end";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order();
tag = "graph_exec_order";
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(exec_order));
#endif
if (save_graphs) {
std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
DumpIR(file_name, kernel_graph, true, kTopStack);

@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -62,6 +62,9 @@
#include "debug/anf_ir_dump.h"
#include "debug/dump_proto.h"
#include "toolchain/adx_datadump_server.h"
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/running_data_recorder.h"
#endif
#if ENABLE_CPU && ENABLE_D
#include "ps/util.h"
#include "ps/ps_cache/ps_cache_manager.h"
@ -129,7 +132,6 @@ std::vector<CNodePtr> GetCNodes(const std::vector<AnfNodePtr> &anf_nodes) {
}
return cnodes;
}
void InsertMakeTupleForOutput(NotNull<KernelGraphPtr> root_graph) {
auto return_node = root_graph->get_return();
MS_EXCEPTION_IF_NULL(return_node);
@ -715,7 +717,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
InitRuntimeResource();
return root_graph->graph_id();
}
// create parameter for multiple branch
std::set<KernelGraphPtr> memo;
CreateMultiBranchOutput(NOT_NULL(root_graph), NOT_NULL(&memo));
@ -726,10 +727,8 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
MultiCallGraphOptimize(NOT_NULL(root_graph));
// resource initialize
InitRuntimeResource();
IrFusionPass(NOT_NULL(root_graph), NOT_NULL(&memo));
memo.clear();
SelectKernel(NOT_NULL(root_graph));
memo.clear();
@ -740,7 +739,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
LoadGraphsToDbg(NOT_NULL(root_graph), NOT_NULL(&memo));
}
memo.clear();
UpdateRefOutputMap(NOT_NULL(root_graph), NOT_NULL(&memo));
memo.clear();
// add make_tuple to the output graph
@ -757,7 +755,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
// adjust kernel
AdjustKernel(root_graph);
// reorder send/recv
auto execution_order = root_graph->execution_order();
ReorderSendRecv(&execution_order);
@ -1333,6 +1330,9 @@ void AscendSession::Execute(const std::shared_ptr<KernelGraph> &kernel_graph, bo
bool ret_ok = runtime_instance->Run(kernel_graph.get(), is_task_sink);
Dump(kernel_graph);
if (!ret_ok) {
#ifdef ENABLE_DUMP_IR
mindspore::RDR::TriggerAll();
#endif
MS_LOG(EXCEPTION) << "run task error!";
}
MS_LOG(INFO) << "Finish!";

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,6 +20,9 @@
#include "runtime/device/kernel_runtime_manager.h"
#include "utils/comm_manager.h"
#include "utils/scoped_long_running.h"
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/running_data_recorder.h"
#endif
#if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@ -114,6 +117,11 @@ void RunGraphTask::Run() {
MS_LOG(ERROR) << "Invalid graph id " << graph_id_;
return;
}
#ifdef ENABLE_DUMP_IR
std::string tag = "run_graph";
std::string file_type = ".ir;.pb";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
#endif
graph->ResetGraphRunningStatus();
try {
session_->RunGraphImpl(graph_id_, input_tensors_, &outputs_);

@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -39,7 +39,9 @@
#include "debug/anf_ir_dump.h"
#include "mindspore/core/base/base_ref_utils.h"
#include "utils/trace_base.h"
#ifdef ENABLE_DUMP_IR
#include "debug/rdr/running_data_recorder.h"
#endif
#if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
#include "ps/ps_cache/ps_cache_manager.h"
#include "ps/common.h"
@ -1089,6 +1091,11 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
auto node_list = TopoSort(func_graph->get_return());
auto graph = NewKernelGraph();
MS_EXCEPTION_IF_NULL(graph);
#ifdef ENABLE_DUMP_IR
std::string tag = "constructed_kernel_graph";
std::string file_type = ".ir;.pb";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
#endif
front_backend_graph_map_[func_graph] = graph;
MS_LOG(INFO) << "Create graph: " << graph->graph_id();
for (const auto &node : node_list) {

@ -5,6 +5,11 @@ set(_DEBUG_SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc"
)
if(ENABLE_DEBUGGER)

@ -0,0 +1,47 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_BASE_RECORDER_H_
#define MINDSPORE_BASE_RECORDER_H_
#include <memory>
#include <string>
namespace mindspore {
class BaseRecorder {
public:
BaseRecorder() : module_(""), tag_(""), directory_("./rdr/"), filename_(""), timestamp_("") {}
BaseRecorder(const std::string &module, const std::string &tag)
: module_(module), tag_(tag), directory_("./rdr/"), filename_(""), timestamp_("") {}
~BaseRecorder() {}
std::string GetModule() { return module_; }
std::string GetTag() { return tag_; }
void SetDirectory(const std::string &directory) { directory_ = directory; }
virtual void Export() {}
protected:
std::string module_;
std::string tag_;
std::string directory_;
std::string filename_;
std::string timestamp_; // year,month,day,hour,minute,second
};
using BaseRecorderPtr = std::shared_ptr<BaseRecorder>;
} // namespace mindspore
#endif // MINDSPORE_BASE_RECORDER_H_

@ -0,0 +1,54 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/rdr/graph_exec_order_recorder.h"
#include <sstream>
#include <fstream>
#include "mindspore/core/ir/anf.h"
#include "mindspore/core/utils/log_adapter.h"
#include "backend/session/anf_runtime_algorithm.h"
namespace mindspore {
namespace {
bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr> &execution_order) {
std::ofstream fout(filename, std::ofstream::app);
if (!fout.is_open()) {
MS_LOG(WARNING) << "Open file for saving graph exec order failed.";
return false;
}
fout << "================== execution order ==================\n";
fout << "execution_order size: " << execution_order.size() << "\n";
int i = 0;
for (auto &cnode : execution_order) {
MS_EXCEPTION_IF_NULL(cnode);
fout << i << ":\n";
fout << "\t" << cnode->DebugString() << "\n";
fout << "\t" << AnfAlgo::GetStreamDistinctionLabel(cnode.get()) << "\n";
fout << "\t" << AnfAlgo::GetGraphId(cnode.get()) << "\n";
i++;
}
fout << "================== execution order ==================\n";
fout.close();
return true;
}
} // namespace
void GraphExecOrderRecorder::Export() {
if (filename_.empty()) {
filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_ + ".txt";
}
DumpGraphExeOrder(filename_, exec_order_);
}
} // namespace mindspore

@ -0,0 +1,42 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef GRAPH_EXEC_ORDER_RENDER_H_
#define GRAPH_EXEC_ORDER_RENDER_H_
#include <vector>
#include <string>
#include <memory>
#include "debug/rdr/base_recorder.h"
namespace mindspore {
class CNode;
using CNodePtr = std::shared_ptr<CNode>;
class GraphExecOrderRecorder : public BaseRecorder {
public:
GraphExecOrderRecorder() : BaseRecorder() {}
GraphExecOrderRecorder(const std::string &module, const std::string &tag,
const std::vector<CNodePtr> &final_exec_order)
: BaseRecorder(module, tag), exec_order_(final_exec_order) {}
void SetModule(const std::string &module) { module_ = module; }
void SetExecOrder(const std::vector<CNodePtr> &final_exec_order) { exec_order_ = final_exec_order; }
virtual void Export();
private:
std::vector<CNodePtr> exec_order_;
};
using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>;
} // namespace mindspore
#endif // GRAPH_EXEC_ORDER_RENDER_H

@ -0,0 +1,46 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/rdr/graph_recorder.h"
#include "mindspore/core/base/base.h"
#include "mindspore/core/ir/func_graph.h"
#include "mindspore/core/utils/log_adapter.h"
#include "debug/anf_ir_dump.h"
#include "debug/anf_ir_utils.h"
#include "debug/dump_proto.h"
namespace mindspore {
void GraphRecorder::Export() {
bool save_flag = false;
if (filename_.empty()) {
filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_;
}
if (graph_type_.find(".dat") != std::string::npos) {
save_flag = true;
ExportIR(filename_ + ".dat", std::to_string(id_), func_graph_); // saving *.dat file
}
if (graph_type_.find(".ir") != std::string::npos) {
save_flag = true;
DumpIR(filename_ + ".ir", func_graph_); // saving *.ir file
}
if (graph_type_.find(".pb") != std::string::npos) {
save_flag = true;
DumpIRProto(func_graph_, filename_); // save *.pb file
}
if (!save_flag) {
MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_;
}
}
} // namespace mindspore

@ -0,0 +1,47 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_GRAPH_RECORDER_H_
#define MINDSPORE_GRAPH_RECORDER_H_
#include <vector>
#include <string>
#include <memory>
#include "debug/rdr/base_recorder.h"
namespace mindspore {
class FuncGraph;
using FuncGraphPtr = std::shared_ptr<FuncGraph>;
class GraphRecorder : public BaseRecorder {
public:
GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {}
GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph,
const std::string &file_type, const int graph_id)
: BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), id_(graph_id) {}
~GraphRecorder() {}
void SetModule(const std::string &module) { module_ = module; }
void SetGraphType(const std::string &file_type) { graph_type_ = file_type; }
void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; }
void SetNodeId(int id) { id_ = id; }
virtual void Export();
private:
FuncGraphPtr func_graph_;
std::string graph_type_;
int id_{0};
};
using GraphRecorderPtr = std::shared_ptr<GraphRecorder>;
} // namespace mindspore
#endif // MINDSPORE_GRAPH_RECORDER_H_

@ -0,0 +1,47 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/rdr/recorder_manager.h"
#include <utility>
#include "debug/rdr/base_recorder.h"
#include "mindspore/core/base/base.h"
#include "mindspore/core/ir/func_graph.h"
namespace mindspore {
bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) {
if (recorder == nullptr) {
MS_LOG(ERROR) << "register recorder module with nullptr.";
return false;
}
std::string module = recorder->GetModule();
std::lock_guard<std::mutex> lock(mtx_);
recorder_container_[module].push_back(std::move(recorder));
return true;
}
void RecorderManager::TriggerAll() {
bool trigger = false;
std::lock_guard<std::mutex> lock(mtx_);
for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) {
for (auto &recorder : iter->second) {
recorder->Export();
trigger = true;
}
}
if (!trigger) {
MS_LOG(WARNING) << "There is no recorder to export.";
}
}
} // namespace mindspore

@ -0,0 +1,49 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_RECORDER_MANAGER_H_
#define MINDSPORE_RECORDER_MANAGER_H_
#include <vector>
#include <string>
#include <sstream>
#include <map>
#include <unordered_map>
#include <memory>
#include <mutex>
namespace mindspore {
class BaseRecorder;
using BaseRecorderPtr = std::shared_ptr<BaseRecorder>;
using BaseRecorderPtrList = std::vector<BaseRecorderPtr>;
class RecorderManager {
public:
static RecorderManager &Instance() {
static RecorderManager manager;
return manager;
}
bool RecordObject(const BaseRecorderPtr &recorder);
void TriggerAll();
private:
RecorderManager() {}
~RecorderManager() {}
mutable std::mutex mtx_;
// module, BaserRecorderPtrList
std::unordered_map<std::string, BaseRecorderPtrList> recorder_container_;
};
} // namespace mindspore
#endif // MINDSPORE_RECORDER_MANAGER_H

@ -0,0 +1,129 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/rdr/running_data_recorder.h"
#include <utility>
#include "debug/rdr/graph_recorder.h"
#include "debug/rdr/somas_recorder.h"
#include "debug/rdr/graph_exec_order_recorder.h"
#include "debug/rdr/recorder_manager.h"
#include "mindspore/core/ir/func_graph.h"
#include "mindspore/core/ir/anf.h"
namespace mindspore {
namespace {
static const char *GetSubModuleName(SubModuleId module_id) {
static const char *sub_module_names[NUM_SUBMODUES] = {
"UNKNOWN", // SM_UNKNOWN
"CORE", // SM_CORE
"ANALYZER", // SM_ANALYZER
"COMMON", // SM_COMMON
"DEBUG", // SM_DEBUG
"DEVICE", // SM_DEVICE
"GE_ADPT", // SM_GE_ADPT
"IR", // SM_IR
"KERNEL", // SM_KERNEL
"MD", // SM_MD
"ME", // SM_ME
"EXPRESS", // SM_EXPRESS
"OPTIMIZER", // SM_OPTIMIZER
"PARALLEL", // SM_PARALLEL
"PARSER", // SM_PARSER
"PIPELINE", // SM_PIPELINE
"PRE_ACT", // SM_PRE_ACT
"PYNATIVE", // SM_PYNATIVE
"SESSION", // SM_SESSION
"UTILS", // SM_UTILS
"VM", // SM_VM
"PROFILER", // SM_PROFILER
"PS", // SM_PS
"LITE", // SM_LITE
"HCCL_ADPT" // SM_HCCL_ADPT
};
return sub_module_names[module_id % NUM_SUBMODUES];
}
} // namespace
namespace RDR {
#ifdef __linux__
bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
const std::string &file_type, int graph_id) {
std::string submodule_name = std::string(GetSubModuleName(module));
GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, tag, graph, file_type, graph_id);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder));
return ans;
}
bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
const std::vector<CNodePtr> &&final_exec_order) {
std::string submodule_name = std::string(GetSubModuleName(module));
GraphExecOrderRecorderPtr graph_exec_order_recorder =
std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder));
return ans;
}
bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
std::string submodule_name = std::string(GetSubModuleName(module));
SomasRecorderPtr somas_recorder = std::make_shared<SomasRecorder>(submodule_name, tag, somas_ptr, graph_id);
somas_recorder->GenString();
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder));
return ans;
}
void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
#else
bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
const std::string &file_type, int graph_id) {
static bool already_printed = false;
std::string submodule_name = std::string(GetSubModuleName(module));
if (already_printed) {
return false;
}
already_printed = true;
MS_LOG(WARNING) << "The RDR presently only support linux os " << submodule_name;
return false;
}
bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std::vector<CNodePtr> &&final_exec_order) {
static bool already_printed = false;
if (already_printed) {
return false;
}
already_printed = true;
MS_LOG(WARNING) << "The RDR presently only support linux os.";
return false;
}
bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
static bool already_printed = false;
if (already_printed) {
return false;
}
already_printed = true;
MS_LOG(WARNING) << "The RDR presently only support linux os.";
return false;
}
void TriggerAll() {
static bool already_printed = false;
if (already_printed) {
return;
}
already_printed = true;
MS_LOG(WARNING) << "The RDR presently only support linux os.";
}
#endif // __linux__
} // namespace RDR
} // namespace mindspore

@ -0,0 +1,40 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef RUNNING_DATA_RECORDER_H_
#define RUNNING_DATA_RECORDER_H_
#include <vector>
#include <string>
#include <memory>
#include "backend/optimizer/somas/somas.h"
#include "mindspore/core/utils/log_adapter.h"
namespace mindspore {
class FuncGraph;
class CNode;
using FuncGraphPtr = std::shared_ptr<FuncGraph>;
using CNodePtr = std::shared_ptr<CNode>;
using SomasPtr = std::shared_ptr<somas::Somas>;
namespace RDR {
bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0);
bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
const std::vector<CNodePtr> &&final_exec_order);
bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id);
void TriggerAll();
} // namespace RDR
} // namespace mindspore
#endif // RUNNING_DATA_RECORDER_H_

@ -0,0 +1,30 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "debug/rdr/somas_recorder.h"
#include "backend/optimizer/somas/somas.h"
namespace mindspore {
void SomasRecorder::Export() {
if (filename_.empty()) {
filename_ = directory_ + module_ + "_" + tag_;
}
std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
somas_reuse_util_ptr_->DumpSomasInfoIR(filename);
std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename);
}
bool SomasRecorder::GenString() { return true; }
} // namespace mindspore

@ -0,0 +1,46 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef SOMAS_RECORDER_H_
#define SOMAS_RECORDER_H_
#include <string>
#include <sstream>
#include <memory>
#include "debug/rdr/base_recorder.h"
namespace mindspore {
namespace somas {
class Somas;
using SomasPtr = std::shared_ptr<Somas>;
} // namespace somas
class SomasRecorder : public BaseRecorder {
public:
SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {}
SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr,
int graph_id)
: BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {}
void SetModule(const std::string &module) { module_ = module; }
void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; }
bool GenString();
virtual void Export();
private:
somas::SomasPtr somas_reuse_util_ptr_;
int graph_id_{0};
};
using SomasRecorderPtr = std::shared_ptr<SomasRecorder>;
} // namespace mindspore
#endif // SOMAS_RECORDER_H
Loading…
Cancel
Save