m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!8545 Save task debug info to ir

Browse Source 
From: @laiyongqiang
Reviewed-by: @kisnwang,@jjfeing
Signed-off-by: @jjfeing
pull/8545/MERGE
mindspore-ci-bot 5 years ago committed by Gitee
parent 3c6b91f6fa 98f7912a67
commit 8612885167
3 changed files with 118 additions and 6 deletions
Show Stats Download Patch File Download Diff File

3
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc
Unescape View File

@ -399,7 +399,8 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) {
#endif
vector<std::shared_ptr<TaskInfo>> task_info_list;
auto anf_node_list = graph->execution_order();
TaskGenerator::GenTasks(anf_node_list, &task_info_list, graph->graph_id());
auto task_generator = TaskGenerator();
task_generator.GenTasks(anf_node_list, &task_info_list, graph->graph_id());
// Store the task_info_list
auto insert_ret = task_map_.insert(std::make_pair(graph->graph_id(), task_info_list));
if (!insert_ret.second) {

96
mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc
Unescape View File

@ -36,6 +36,14 @@ bool TaskGenerator::GenTasks(const std::vector<CNodePtr> &anf_node_list, std::ve
return false;
}
MS_LOG(INFO) << "GenTasks end...";
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH);
if (save_graphs) {
std::string file_path = save_graphs_path + "/" + "task_info" + "_graph_" + std::to_string(graph_id) + ".ir";
DumpTaskInfo(file_path);
}
return true;
}
@ -166,6 +174,15 @@ bool TaskGenerator::LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_i
std::vector<TaskInfoPtr> task_info_ptrs =
ascend_kernel_mod->GenTask(kernel_inputs, kernel_workspaces, kernel_outputs, stream_id);
task_info_list->insert(task_info_list->end(), task_info_ptrs.begin(), task_info_ptrs.end());
auto debug_info = std::make_shared<TaskDebugInfo>();
debug_info->op_name_ = anf_node_ptr->fullname_with_scope();
debug_info->task_num_ = task_info_ptrs.size();
debug_info->stream_id_ = task_info_ptrs[0]->stream_id();
debug_info->dump_flag_ = task_info_ptrs[0]->dump_flag();
debug_info->input_addrs_ = kernel_inputs;
debug_info->output_addrs_ = kernel_outputs;
debug_info->workspace_addrs_ = kernel_workspaces;
task_debug_info_list_.push_back(debug_info);
return true;
}
@ -195,8 +212,87 @@ bool TaskGenerator::LaunchAllKernel(const std::vector<CNodePtr> &anf_node_list,
if (ProfilingManager::GetInstance().IsProfiling()) {
ProfilingUtils::SetGraphProfilingCNode(graph_id, profiling_cnode_list);
}
return true;
}
#ifdef ENABLE_DUMP_IR
void TaskGenerator::DumpTaskInfo(const std::string &real_filename) {
if (real_filename.size() > PATH_MAX) {
MS_LOG(ERROR) << "File path " << real_filename << " is too long.";
return;
}
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
}
#else
if (nullptr == realpath(real_filename.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << real_filename << " does not exit.";
}
#endif
OrderedMap<AnfNodePtr, int32_t> para_map;
std::string path_string = real_path;
ChangeFileMode(path_string, S_IRWXU);
std::ofstream fout(real_path);
if (!fout.is_open()) {
MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!";
return;
}
size_t index = 0;
for (auto &task_debug_info : task_debug_info_list_) {
fout << "op_name:" << task_debug_info->op_name_ << "\n"
<< "task_index:" << index << "\t"
<< "task_num:" << task_debug_info->task_num_ << "\t"
<< "task0_stream_id:" << task_debug_info->stream_id_ << "\t"
<< "task0_type:" << task_debug_info->type_ << "\t"
<< "task0_dump_flag:" << task_debug_info->dump_flag_ << "\n";
index++;
if (task_debug_info->input_addrs_.size()) {
fout << "input address:";
for (auto &input : task_debug_info->input_addrs_) {
fout << input->addr << "(" << input->size << ")\t";
}
fout << "\n";
}
if (task_debug_info->output_addrs_.size()) {
fout << "output address:";
for (auto &output : task_debug_info->output_addrs_) {
fout << output->addr << "(" << output->size << ")\t";
}
fout << "\n";
}
if (task_debug_info->workspace_addrs_.size()) {
fout << "workspace address:";
for (auto &workspace : task_debug_info->workspace_addrs_) {
fout << workspace->addr << "(" << workspace->size << ")\t";
}
fout << "\n";
}
fout << "\n";
}
fout.close();
// set file mode to read only by user
ChangeFileMode(path_string, S_IRUSR);
}
#else
void TaskGenerator::DumpTaskInfo(const std::string &real_filename) {
static bool already_printed = false;
if (already_printed) {
return;
}
already_printed = true;
MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, "
<< "please recompile source to enable it. See help of building script.";
}
#endif
} // namespace tasksink
} // namespace ascend
} // namespace device

25
mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h
Unescape View File

@ -36,6 +36,19 @@ using mindspore::kernel::AddressPtr;
using AddressPtrList = std::vector<mindspore::kernel::AddressPtr>;
using ge::model_runner::TaskInfo;
using TaskInfoPtr = std::shared_ptr<TaskInfo>;
class TaskDebugInfo {
public:
std::string op_name_;
std::size_t task_num_{0};
uint32_t stream_id_{0};
uint32_t type_{0};
bool dump_flag_{false};
std::vector<AddressPtr> input_addrs_;
std::vector<AddressPtr> output_addrs_;
std::vector<AddressPtr> workspace_addrs_;
};
using TaskDebugInfoPtr = std::shared_ptr<TaskDebugInfo>;
class TaskGenerator {
public:
TaskGenerator() = default;
@ -43,15 +56,17 @@ class TaskGenerator {
TaskGenerator(const TaskGenerator &in) = delete;
TaskGenerator &operator=(const TaskGenerator &in) = delete;
static bool GenTasks(const std::vector<CNodePtr> &anf_node_list, std::vector<TaskInfoPtr> *task_info_list,
uint32_t graph_id);
bool GenTasks(const std::vector<CNodePtr> &anf_node_list, std::vector<TaskInfoPtr> *task_info_list,
uint32_t graph_id);
private:
std::vector<TaskDebugInfoPtr> task_debug_info_list_;
static void LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs);
static void LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs);
static bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector<TaskInfoPtr> *task_info_list);
static bool LaunchAllKernel(const std::vector<CNodePtr> &anf_node_list, std::vector<TaskInfoPtr> *task_info_list,
uint32_t graph_id);
bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector<TaskInfoPtr> *task_info_list);
bool LaunchAllKernel(const std::vector<CNodePtr> &anf_node_list, std::vector<TaskInfoPtr> *task_info_list,
uint32_t graph_id);
void DumpTaskInfo(const string &real_filename);
};
} // namespace tasksink
} // namespace ascend

Write Preview
Loading…
Cancel
Save