m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implementation for dumping const values for each step

Browse Source 
fix no output address error

address clang-format error

address some comments

clang_format check
pull/7269/head
lichen_101010 4 years ago
parent a75665ce49
commit 678962d718
6 changed files with 111 additions and 50 deletions
Show Stats Download Patch File Download Diff File

59
mindspore/ccsrc/debug/data_dump/e2e_dump_util.cc
Unescape View File

@ -28,7 +28,8 @@
namespace {
const size_t PRAMATER_OUTPUT_INDEX = 0;
}
const size_t VALUE_NODE_OUTPUT_INDEX = 0;
} // namespace
namespace mindspore {
void E2eDumpUtil::GetFileKernelName(NotNull<std::string *> kernel_name) {
@ -163,33 +164,53 @@ void E2eDumpUtil::DumpInput(const session::KernelGraph *graph, const std::string
}
}
void E2eDumpUtil::DumpParameters(const session::KernelGraph *graph, const std::string &dump_path, Debugger *debugger) {
MS_EXCEPTION_IF_NULL(graph);
void E2eDumpUtil::DumpSingleAnfnode(const AnfNodePtr &anf_node, const size_t output_index, const std::string &dump_path,
bool trans_flag, Debugger *debugger) {
MS_EXCEPTION_IF_NULL(anf_node);
auto &dump_json_parser = DumpJsonParser::GetInstance();
MS_LOG(INFO) << "Start e2e dump parameters";
bool trans_flag = dump_json_parser.trans_flag();
const auto &parameters = graph->inputs();
for (auto &item : parameters) {
if (!item->isa<Parameter>()) {
continue;
if (!anf_node->isa<Parameter>() && !anf_node->isa<ValueNode>()) {
return;
}
std::string parameter_name = item->fullname_with_scope();
if (!dump_json_parser.NeedDump(parameter_name)) {
continue;
std::string node_name = anf_node->fullname_with_scope();
if (!dump_json_parser.NeedDump(node_name)) {
return;
}
DumpJsonParser::GetInstance().MatchKernel(parameter_name);
auto addr = AnfAlgo::GetOutputAddr(item, PRAMATER_OUTPUT_INDEX);
DumpJsonParser::GetInstance().MatchKernel(node_name);
GetFileKernelName(NOT_NULL(&node_name));
// check if output adde exists, if not, return;
if (!AnfAlgo::OutputAddrExist(anf_node, output_index)) {
return;
}
auto addr = AnfAlgo::GetOutputAddr(anf_node, output_index);
MS_EXCEPTION_IF_NULL(addr);
ShapeVector int_shapes;
GetDumpIntShape(item, PRAMATER_OUTPUT_INDEX, trans_flag, NOT_NULL(&int_shapes));
auto type = AnfAlgo::GetOutputInferDataType(item, PRAMATER_OUTPUT_INDEX);
GetDumpIntShape(anf_node, output_index, trans_flag, NOT_NULL(&int_shapes));
auto type = AnfAlgo::GetOutputInferDataType(anf_node, output_index);
std::string file_path = dump_path + '/' + parameter_name + '_' + "output_0";
std::string file_path = dump_path + '/' + node_name + '_' + "output_0";
if (IsDeviceTargetGPU()) {
DumpGPUMemToFile(file_path, parameter_name, NOT_NULL(addr), trans_flag, int_shapes, type, 0, debugger);
DumpGPUMemToFile(file_path, node_name, NOT_NULL(addr), trans_flag, int_shapes, type, 0, debugger);
} else {
DumpMemToFile(file_path, NOT_NULL(addr), trans_flag, int_shapes, type);
}
}
void E2eDumpUtil::DumpParametersAndConst(const session::KernelGraph *graph, const std::string &dump_path,
Debugger *debugger) {
MS_EXCEPTION_IF_NULL(graph);
auto &dump_json_parser = DumpJsonParser::GetInstance();
MS_LOG(INFO) << "Start e2e dump parameters and Const values";
bool trans_flag = dump_json_parser.trans_flag();
// dump parameters
const auto &parameters = graph->inputs();
for (auto &item : parameters) {
DumpSingleAnfnode(item, PRAMATER_OUTPUT_INDEX, dump_path, trans_flag, debugger);
}
// dump const values
auto value_nodes = graph->graph_value_nodes();
for (const auto &value_node : value_nodes) {
DumpSingleAnfnode(value_node, VALUE_NODE_OUTPUT_INDEX, dump_path, trans_flag, debugger);
}
}
bool E2eDumpUtil::DumpData(const session::KernelGraph *graph, uint32_t device_id, Debugger *debugger) {
@ -218,7 +239,7 @@ bool E2eDumpUtil::DumpData(const session::KernelGraph *graph, uint32_t device_id
dump_path += (net_name + "/device_" + std::to_string(device_id) + "/iteration_" + iterator);
DumpInput(graph, dump_path, debugger);
DumpOutput(graph, dump_path, debugger);
DumpParameters(graph, dump_path, debugger);
DumpParametersAndConst(graph, dump_path, debugger);
return true;
}
} // namespace mindspore

8
mindspore/ccsrc/debug/data_dump/e2e_dump_util.h
Unescape View File

@ -20,6 +20,7 @@
#include <string>
#include "backend/session/kernel_graph.h"
#include "runtime/device/device_address.h"
#include "debug/data_dump/dump_json_parser.h"
#ifndef ENABLE_DEBUGGER
class Debugger;
#endif
@ -29,13 +30,14 @@ class E2eDumpUtil {
E2eDumpUtil() = default;
~E2eDumpUtil() = default;
static bool DumpData(const session::KernelGraph *graph, uint32_t device_id, Debugger *debugger = nullptr);
static void GetFileKernelName(NotNull<std::string *> kernel_name);
private:
static void DumpOutput(const session::KernelGraph *graph, const std::string &dump_path, Debugger *debugger);
static void DumpInput(const session::KernelGraph *graph, const std::string &dump_path, Debugger *debugger);
static void DumpParameters(const session::KernelGraph *graph, const std::string &dump_path, Debugger *debugger);
static void DumpParametersAndConst(const session::KernelGraph *graph, const std::string &dump_path,
Debugger *debugger);
static void GetFileKernelName(NotNull<std::string *> kernel_name);
static void DumpMemToFile(const std::string &file_path, NotNull<const device::DeviceAddress *> addr, bool trans_flag,
const ShapeVector &int_shapes, const TypeId &type);
static void DumpGPUMemToFile(const std::string &file_path, const std::string &original_kernel_name,
@ -43,6 +45,8 @@ class E2eDumpUtil {
const ShapeVector &int_shapes, const TypeId &type, size_t slot, Debugger *debugger);
static void GetDumpIntShape(const AnfNodePtr &node, size_t index, bool trans_flag, NotNull<ShapeVector *> int_shapes);
static bool IsDeviceTargetGPU();
static void DumpSingleAnfnode(const AnfNodePtr &anf_node, const size_t output_index, const std::string &dump_path,
bool trans_flag, Debugger *debugger);
};
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_DEBUG_DATA_DUMP_E_2_E_DUMP_UTIL_H_

70
mindspore/ccsrc/debug/debugger/debugger.cc
Unescape View File

@ -31,6 +31,7 @@
#include "backend/session/anf_runtime_algorithm.h"
#include "runtime/device/kernel_runtime_manager.h"
#include "runtime/device/kernel_runtime.h"
#include "debug/data_dump/e2e_dump_util.h"
using debugger::EventReply;
using debugger::GraphProto;
@ -49,6 +50,7 @@ namespace mindspore {
DebuggerPtr Debugger::debugger_ = nullptr;
std::mutex Debugger::instance_lock_;
static const size_t PRAMATER_OUTPUT_INDEX = 0;
static const size_t VALUE_NODE_OUTPUT_INDEX = 0;
Debugger::Debugger()
: grpc_client_(nullptr),
@ -254,6 +256,9 @@ void Debugger::PreExecute(const KernelGraphPtr &graph_ptr) {
void Debugger::PostExecute() {
// access lock for public method
std::lock_guard<std::mutex> a_lock(access_lock_);
if (pipeline::ExecutorPy::GetDebugTerminate()) {
return;
}
if (debugger_->DebuggerBackendEnabled()) {
// analyze tensor data and send the watchpoints been hit
if (run_level_ == "node") {
@ -287,6 +292,9 @@ bool Debugger::ReadNodeDataRequired() {
void Debugger::PostExecuteNode() {
// access lock for public method
std::lock_guard<std::mutex> a_lock(access_lock_);
if (pipeline::ExecutorPy::GetDebugTerminate()) {
return;
}
if (debugger_enabled_ && !is_dataset_graph_) {
auto watchpoint_table = debug_services_->GetWatchpointTable();
auto is_watchpoint = debug_services_->IsWatchPoint(cur_name_, watchpoint_table);
@ -333,7 +341,7 @@ void Debugger::CheckGraphPtr(const KernelGraphPtr &graph_ptr) {
// only try to enable debugger if it is not a dataset graph
EnableDebugger();
if (debugger_enabled_) {
LoadParameters();
LoadParametersAndConst();
// get graph proto and send to mindinsight
SendGraphAndSuspend(GetGraphProto());
}
@ -601,7 +609,7 @@ void Debugger::Exit() {
if (run_level_ == "node") {
pipeline::ClearResAtexit();
exit(1);
} else if (run_level_ == "step") {
} else if (run_level_ == "step" || device_target_ == kAscendDevice) {
// Notify main thread to terminate
pipeline::ExecutorPy::DebugTerminate(true);
} else {
@ -861,34 +869,60 @@ bool Debugger::CheckPort(const char *port) {
return true;
}
void Debugger::LoadParameters() {
if (!(debugger_enabled_ || CheckDebuggerDumpEnabled())) return;
if (!(num_step_ == 0 || device_target_ == kAscendDevice ||
(device_target_ == kGPUDevice && device::KernelRuntime::DumpDataEnabledIteration())))
void Debugger::LoadSingleAnfnode(const AnfNodePtr &anf_node, const size_t output_index) {
MS_EXCEPTION_IF_NULL(anf_node);
if (!anf_node->isa<Parameter>() && !anf_node->isa<ValueNode>()) {
return;
MS_EXCEPTION_IF_NULL(graph_ptr_);
const auto &parameters = graph_ptr_->inputs();
// for parameters, set its execution order to be 0;
}
bool keep_prev;
if (anf_node->isa<Parameter>()) {
keep_prev = true;
} else {
keep_prev = false;
}
// for parameters and value nodes, set its execution order to be 0;
int exec_order = 0;
for (auto &item : parameters) {
if (!item->isa<Parameter>()) {
continue;
std::string node_name = anf_node->fullname_with_scope();
E2eDumpUtil::GetFileKernelName(NOT_NULL(&node_name));
// check if output adde exists, if not, return;
if (!AnfAlgo::OutputAddrExist(anf_node, output_index)) {
return;
}
std::string parameter_name = item->fullname_with_scope();
auto addr = AnfAlgo::GetOutputAddr(item, PRAMATER_OUTPUT_INDEX);
auto type = AnfAlgo::GetOutputInferDataType(item, PRAMATER_OUTPUT_INDEX);
auto addr = AnfAlgo::GetOutputAddr(anf_node, output_index);
MS_EXCEPTION_IF_NULL(addr);
auto type = AnfAlgo::GetOutputInferDataType(anf_node, output_index);
auto format = kOpFormat_DEFAULT;
string tensor_name = parameter_name + ':' + "0";
string tensor_name = node_name + ':' + "0";
ShapeVector int_shapes;
auto shape = AnfAlgo::GetOutputDeviceShape(item, PRAMATER_OUTPUT_INDEX);
auto shape = AnfAlgo::GetOutputDeviceShape(anf_node, output_index);
(void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes),
[](size_t inner_item) { return SizeToInt(inner_item); });
bool ret = addr->LoadMemToHost(tensor_name, exec_order, format, int_shapes, type, 0, true);
bool ret = addr->LoadMemToHost(tensor_name, exec_order, format, int_shapes, type, 0, keep_prev);
if (!ret) {
MS_LOG(ERROR) << "LoadMemToHost:"
<< ", tensor_name:" << tensor_name << ", host_format:" << format << ".!";
}
}
void Debugger::LoadParametersAndConst() {
if (!(debugger_enabled_ || CheckDebuggerDumpEnabled())) return;
if (!(num_step_ == 0 || device_target_ == kAscendDevice ||
(device_target_ == kGPUDevice && device::KernelRuntime::DumpDataEnabledIteration())))
return;
MS_EXCEPTION_IF_NULL(graph_ptr_);
// load parameters
MS_LOG(INFO) << "Start to load Parameters!";
const auto &parameters = graph_ptr_->inputs();
for (auto &item : parameters) {
LoadSingleAnfnode(item, PRAMATER_OUTPUT_INDEX);
}
// load value nodes
// get all constant avlues from the graph
MS_LOG(INFO) << "Start to load value nodes!";
const auto value_nodes = graph_ptr_->graph_value_nodes();
for (auto &item : value_nodes) {
LoadSingleAnfnode(item, VALUE_NODE_OUTPUT_INDEX);
}
}
} // namespace mindspore

4
mindspore/ccsrc/debug/debugger/debugger.h
Unescape View File

@ -103,7 +103,7 @@ class Debugger : public std::enable_shared_from_this<Debugger> {
void SendMetadata();
void LoadParameters();
void LoadParametersAndConst();
private:
// private constructor for singleton
@ -164,6 +164,8 @@ class Debugger : public std::enable_shared_from_this<Debugger> {
// Check if the port is valid
bool CheckPort(const char *port);
void LoadSingleAnfnode(const AnfNodePtr &anf_node, const size_t output_index);
// class members
std::unique_ptr<GrpcClient> grpc_client_;
std::unique_ptr<DebugServices> debug_services_;

2
mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc
Unescape View File

@ -327,7 +327,7 @@ bool AscendKernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debug
// load output
LoadOutput(graph, debugger);
// load parameters
if (debugger) debugger->LoadParameters();
if (debugger) debugger->LoadParametersAndConst();
#endif
return true;
}

2
mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc
Unescape View File

@ -650,7 +650,7 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, De
}
if (!mock) {
// collect weights and bias for dump mode
if (debugger) debugger->LoadParameters();
if (debugger) debugger->LoadParametersAndConst();
CHECK_OP_RET_WITH_EXCEPT(SyncStream(), "SyncStream failed.");
}
ClearSwapInfo(mock);

Write Preview
Loading…
Cancel
Save